On Thu, Sep 25, 2008 at 9:23 AM, Matthew Toseland
<toad at amphibian.dyndns.org> wrote:
> On Thursday 25 September 2008 01:09, Daniel Cheng wrote:
>> On Thu, Sep 25, 2008 at 3:03 AM, Matthew Toseland
>> <toad at amphibian.dyndns.org> wrote:
>> > On Wednesday 24 September 2008 05:51, j16sdiz at freenetproject.org wrote:
>> >> Author: j16sdiz
>> >> Date: 2008-09-24 04:51:56 +0000 (Wed, 24 Sep 2008)
>> >> New Revision: 22789
>> >>
>> >> Modified:
>> >> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
>> >> trunk/freenet/src/freenet/node/Node.java
>> >> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
>> >> Log:
>> >> preallocate space for SaltedHashFreenetStore
>> >
>> > Is this the right place to do it? When exactly will it happen? During
>> > migration (offline), during startup (offline), when the size is increased
>> > from config (online) ??
>>
>> it preallocate space for all size increase cases.
>
> Including migration and new store?
yes.
> Is this always during startup?
no, not always.
>>
>> >>
>> >> Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
>> >> ===================================================================
>> >> --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-23
>> > 23:46:32 UTC (rev 22788)
>> >> +++ trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-24
>> > 04:51:56 UTC (rev 22789)
>> >> @@ -710,6 +710,8 @@
>> >> Node.storeDirectory=Store directory
>> >> Node.storeDirectoryLong=Name of directory to put store files in
>> >> Node.storeMaxMemTooHigh=Giving more than 80% of your ram to BDB is
> probably
>> > not what you want to do!
>> >> +Node.storePreallocate=Preallocate space for datastore
>> >> +Node.storePreallocateLong=Preallocate space for datastore
>> >> Node.storeSize=Store size in bytes
>> >> Node.storeSizeLong=Store size in bytes
>> >> Node.storeType=Store type (LEAVE THIS ALONE)
>> >>
>> >> Modified: trunk/freenet/src/freenet/node/Node.java
>> >> ===================================================================
>> >> --- trunk/freenet/src/freenet/node/Node.java 2008-09-23 23:46:32 UTC
> (rev
>> > 22788)
>> >> +++ trunk/freenet/src/freenet/node/Node.java 2008-09-24 04:51:56 UTC
> (rev
>> > 22789)
>> >> @@ -505,6 +505,8 @@
>> >> private static final int MIN_UPTIME_STORE_KEY = 40;
>> >>
>> >> private volatile boolean isPRNGReady = false;
>> >> +
>> >> + private boolean storePreallocate;
>> >>
>> >> /**
>> >> * Read all storable settings (identity etc) from the node file.
>> >> @@ -1604,6 +1606,28 @@
>> >> maxStoreKeys = maxTotalKeys / 2;
>> >> maxCacheKeys = maxTotalKeys - maxStoreKeys;
>> >>
>> >> + nodeConfig.register("storePreallocate", true, sortOrder++,
> true,
>> > false, "Node.storePreallocate", "Node.storePreallocateLong",
>> >> + new BooleanCallback() {
>> >> + @Override
>> >> + public Boolean get() {
>> >> + return storePreallocate;
>> >> + }
>> >> +
>> >> + @Override
>> >> + public void set(Boolean val) throws
>> > InvalidConfigValueException, NodeNeedRestartException {
>> >> + storePreallocate = val;
>> >> + if
> (storeType.equals("salt-hash")) {
>> >> +
> ((SaltedHashFreenetStore)
>> > chkDatastore.getStore()).setPreallocate(val);
>> >> +
> ((SaltedHashFreenetStore)
>> > chkDatacache.getStore()).setPreallocate(val);
>> >> +
> ((SaltedHashFreenetStore)
>> > pubKeyDatastore.getStore()).setPreallocate(val);
>> >> +
> ((SaltedHashFreenetStore)
>> > pubKeyDatacache.getStore()).setPreallocate(val);
>> >> +
> ((SaltedHashFreenetStore)
>> > sskDatastore.getStore()).setPreallocate(val);
>> >> +
> ((SaltedHashFreenetStore)
>> > sskDatacache.getStore()).setPreallocate(val);
>> >> + }
>> >> + }}
>> >> + );
>> >> + storePreallocate =
> nodeConfig.getBoolean("storePreallocate");
>> >> +
>> >> if (storeType.equals("salt-hash")) {
>> >> initSaltHashFS(suffix);
>> >> } else if (storeType.equals("bdb-index")) {
>> >> @@ -1744,32 +1768,32 @@
>> >> System.out.println("Initializing CHK Datastore (" +
> maxStoreKeys + "
>> > keys)");
>> >> chkDatastore = new CHKStore();
>> >> SaltedHashFreenetStore chkDataFS =
>> > SaltedHashFreenetStore.construct(storeDir, "CHK-store", chkDatastore,
>> >> - random, maxStoreKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook);
>> >> + random, maxStoreKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook, storePreallocate);
>> >> Logger.normal(this, "Initializing CHK Datacache");
>> >> System.out.println("Initializing CHK Datacache (" +
> maxCacheKeys + ':' +
>> > maxCacheKeys + " keys)");
>> >> chkDatacache = new CHKStore();
>> >> SaltedHashFreenetStore chkCacheFS =
>> > SaltedHashFreenetStore.construct(storeDir, "CHK-cache", chkDatacache,
>> >> - random, maxCacheKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook);
>> >> + random, maxCacheKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook, storePreallocate);
>> >> Logger.normal(this, "Initializing pubKey
> Datastore");
>> >> System.out.println("Initializing pubKey
> Datastore");
>> >> pubKeyDatastore = new PubkeyStore();
>> >> SaltedHashFreenetStore pubkeyDataFS =
>> > SaltedHashFreenetStore.construct(storeDir, "PUBKEY-store",
>> >> - pubKeyDatastore, random, maxStoreKeys,
> bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook);
>> >> + pubKeyDatastore, random, maxStoreKeys,
> bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook, storePreallocate);
>> >> Logger.normal(this, "Initializing pubKey
> Datacache");
>> >> System.out.println("Initializing pubKey Datacache
> (" + maxCacheKeys + "
>> > keys)");
>> >> pubKeyDatacache = new PubkeyStore();
>> >> SaltedHashFreenetStore pubkeyCacheFS =
>> > SaltedHashFreenetStore.construct(storeDir, "PUBKEY-cache",
>> >> - pubKeyDatacache, random, maxCacheKeys,
> bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook);
>> >> + pubKeyDatacache, random, maxCacheKeys,
> bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook, storePreallocate);
>> >> Logger.normal(this, "Initializing SSK Datastore");
>> >> System.out.println("Initializing SSK Datastore");
>> >> sskDatastore = new SSKStore(this);
>> >> SaltedHashFreenetStore sskDataFS =
>> > SaltedHashFreenetStore.construct(storeDir, "SSK-store", sskDatastore,
>> >> - random, maxStoreKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook);
>> >> + random, maxStoreKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook, storePreallocate);
>> >> Logger.normal(this, "Initializing SSK Datacache");
>> >> System.out.println("Initializing SSK Datacache (" +
> maxCacheKeys + "
>> > keys)");
>> >> sskDatacache = new SSKStore(this);
>> >> SaltedHashFreenetStore sskCacheFS =
>> > SaltedHashFreenetStore.construct(storeDir, "SSK-cache", sskDatacache,
>> >> - random, maxCacheKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook);
>> >> + random, maxCacheKeys, bloomFilterSizeInM,
>> > storeBloomFilterCounting, shutdownHook, storePreallocate);
>> >>
>> >> File migrationFile = new
> File(storeDir, "migrated");
>> >> if (!migrationFile.exists()) {
>> >>
>> >> Modified:
>> > trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
>> >> ===================================================================
>> >> ---
> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
>> > 2008-09-23 23:46:32 UTC (rev 22788)
>> >> +++
> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
>> > 2008-09-24 04:51:56 UTC (rev 22789)
>> >> @@ -79,16 +79,18 @@
>> >> private long storeSize;
>> >> private int generation;
>> >> private int flags;
>> >> +
>> >> + private boolean preallocate = true;
>> >>
>> >> public static SaltedHashFreenetStore construct(File baseDir, String
> name,
>> > StoreCallback callback, Random random,
>> >> - long maxKeys, int bloomFilterSize, boolean bloomCounting,
>> > SemiOrderedShutdownHook shutdownHook)
>> >> + long maxKeys, int bloomFilterSize, boolean bloomCounting,
>> > SemiOrderedShutdownHook shutdownHook, boolean preallocate)
>> >> throws IOException {
>> >> return new SaltedHashFreenetStore(baseDir, name, callback,
> random,
>> > maxKeys, bloomFilterSize, bloomCounting,
>> >> - shutdownHook);
>> >> + shutdownHook, preallocate);
>> >> }
>> >>
>> >> private SaltedHashFreenetStore(File baseDir, String name,
> StoreCallback
>> > callback, Random random, long maxKeys,
>> >> - int bloomFilterSize, boolean bloomCounting,
>> > SemiOrderedShutdownHook shutdownHook) throws IOException {
>> >> + int bloomFilterSize, boolean bloomCounting,
>> > SemiOrderedShutdownHook shutdownHook, boolean preallocate) throws
> IOException
>> > {
>> >> logMINOR = Logger.shouldLog(Logger.MINOR, this);
>> >> logDEBUG = Logger.shouldLog(Logger.DEBUG, this);
>> >>
>> >> @@ -107,6 +109,7 @@
>> >> this.random = random;
>> >> storeSize = maxKeys;
>> >> this.bloomFilterSize = bloomFilterSize;
>> >> + this.preallocate = preallocate;
>> >>
>> >> lockManager = new LockManager();
>> >>
>> >> @@ -746,14 +749,54 @@
>> >> }
>> >>
>> >> /**
>> >> + * Set preallocate storage space
>> >> + * @param preallocate
>> >> + */
>> >> + public void setPreallocate(boolean preallocate) {
>> >> + this.preallocate = preallocate;
>> >> + }
>> >> +
>> >> + /**
>> >> * Change on disk store file size
>> >> *
>> >> * @param storeFileSize
>> >> */
>> >> private void setStoreFileSize(long storeFileSize) {
>> >> try {
>> >> - metaRAF.setLength(Entry.METADATA_LENGTH *
> storeFileSize);
>> >> - hdRAF.setLength((headerBlockLength +
> dataBlockLength + hdPadding) *
>> > storeFileSize);
>> >> + long oldMetaLen = metaRAF.length();
>> >> + long oldHdLen = hdRAF.length();
>> >> +
>> >> + final long newMetaLen = Entry.METADATA_LENGTH *
> storeFileSize;
>> >> + final long newHdLen = (headerBlockLength +
> dataBlockLength + hdPadding)
>> > * storeFileSize;
>> >> +
>> >> + if (preallocate) {
>> >> + byte[] b = new byte[4096];
>> >> + ByteBuffer bf = ByteBuffer.wrap(b);
>> >> +
>> >> + // start from next 4KB boundary => align to
> x86 page size
>> >> + if (oldMetaLen % 4096 != 0)
>> >> + oldMetaLen += 4096 - (oldMetaLen %
> 4096);
>> >> + if (oldHdLen % 4096 != 0)
>> >> + oldHdLen += 4096 - (oldHdLen %
> 4096);
>> >> +
>> >> + // this may write excess the size, the
> setLength() would fix it
>> >> + while (oldMetaLen < newMetaLen) {
>> >> + // never write random byte to meta
> data!
>> >> + // this would screw up the isFree()
> function
>> >> + bf.rewind();
>> >> + metaFC.write(bf, oldMetaLen);
>> >> + oldMetaLen += 4096;
>> >> + }
>> >> + while (oldHdLen < newHdLen) {
>> >> + random.nextBytes(b);
>> >> + bf.rewind();
>> >> + hdFC.write(bf, oldHdLen);
>> >> + oldHdLen += 4096;
>> >> + }
>> >> + }
>> >> +
>> >> + metaRAF.setLength(newMetaLen);
>> >> + hdRAF.setLength(newHdLen);
>> >> } catch (IOException e) {
>> >> Logger.error(this, "error resizing store file", e);
>> >> }
>> >> @@ -1686,5 +1729,5 @@
>> >> } finally {
>> >> configLock.readLock().unlock();
>> >> }
>> >> - }
>> >> + }
>> >> }
>> >>
>> >> _______________________________________________
>> >> cvs mailing list
>> >> cvs at freenetproject.org
>> >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>> >>
>> >>
>> >
>> > _______________________________________________
>> > Devl mailing list
>> > Devl at freenetproject.org
>> > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
>> >
>> _______________________________________________
>> Devl mailing list
>> Devl at freenetproject.org
>> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
>>
>>
>
> _______________________________________________
> Devl mailing list
> Devl at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
>
