On Thursday 25 September 2008 01:09, Daniel Cheng wrote:
> On Thu, Sep 25, 2008 at 3:03 AM, Matthew Toseland
> <toad at amphibian.dyndns.org> wrote:
> > On Wednesday 24 September 2008 05:51, j16sdiz at freenetproject.org wrote:
> >> Author: j16sdiz
> >> Date: 2008-09-24 04:51:56 +0000 (Wed, 24 Sep 2008)
> >> New Revision: 22789
> >>
> >> Modified:
> >> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
> >> trunk/freenet/src/freenet/node/Node.java
> >> trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> >> Log:
> >> preallocate space for SaltedHashFreenetStore
> >
> > Is this the right place to do it? When exactly will it happen? During
> > migration (offline), during startup (offline), when the size is increased
> > from config (online) ??
>
> it preallocate space for all size increase cases.
Including migration and new store?
Is this always during startup?
>
> >>
> >> Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
> >> ===================================================================
> >> --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-23
> > 23:46:32 UTC (rev 22788)
> >> +++ trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-09-24
> > 04:51:56 UTC (rev 22789)
> >> @@ -710,6 +710,8 @@
> >> Node.storeDirectory=Store directory
> >> Node.storeDirectoryLong=Name of directory to put store files in
> >> Node.storeMaxMemTooHigh=Giving more than 80% of your ram to BDB is
probably
> > not what you want to do!
> >> +Node.storePreallocate=Preallocate space for datastore
> >> +Node.storePreallocateLong=Preallocate space for datastore
> >> Node.storeSize=Store size in bytes
> >> Node.storeSizeLong=Store size in bytes
> >> Node.storeType=Store type (LEAVE THIS ALONE)
> >>
> >> Modified: trunk/freenet/src/freenet/node/Node.java
> >> ===================================================================
> >> --- trunk/freenet/src/freenet/node/Node.java 2008-09-23 23:46:32 UTC
(rev
> > 22788)
> >> +++ trunk/freenet/src/freenet/node/Node.java 2008-09-24 04:51:56 UTC
(rev
> > 22789)
> >> @@ -505,6 +505,8 @@
> >> private static final int MIN_UPTIME_STORE_KEY = 40;
> >>
> >> private volatile boolean isPRNGReady = false;
> >> +
> >> + private boolean storePreallocate;
> >>
> >> /**
> >> * Read all storable settings (identity etc) from the node file.
> >> @@ -1604,6 +1606,28 @@
> >> maxStoreKeys = maxTotalKeys / 2;
> >> maxCacheKeys = maxTotalKeys - maxStoreKeys;
> >>
> >> + nodeConfig.register("storePreallocate", true, sortOrder++,
true,
> > false, "Node.storePreallocate", "Node.storePreallocateLong",
> >> + new BooleanCallback() {
> >> + @Override
> >> + public Boolean get() {
> >> + return storePreallocate;
> >> + }
> >> +
> >> + @Override
> >> + public void set(Boolean val) throws
> > InvalidConfigValueException, NodeNeedRestartException {
> >> + storePreallocate = val;
> >> + if
(storeType.equals("salt-hash")) {
> >> +
((SaltedHashFreenetStore)
> > chkDatastore.getStore()).setPreallocate(val);
> >> +
((SaltedHashFreenetStore)
> > chkDatacache.getStore()).setPreallocate(val);
> >> +
((SaltedHashFreenetStore)
> > pubKeyDatastore.getStore()).setPreallocate(val);
> >> +
((SaltedHashFreenetStore)
> > pubKeyDatacache.getStore()).setPreallocate(val);
> >> +
((SaltedHashFreenetStore)
> > sskDatastore.getStore()).setPreallocate(val);
> >> +
((SaltedHashFreenetStore)
> > sskDatacache.getStore()).setPreallocate(val);
> >> + }
> >> + }}
> >> + );
> >> + storePreallocate =
nodeConfig.getBoolean("storePreallocate");
> >> +
> >> if (storeType.equals("salt-hash")) {
> >> initSaltHashFS(suffix);
> >> } else if (storeType.equals("bdb-index")) {
> >> @@ -1744,32 +1768,32 @@
> >> System.out.println("Initializing CHK Datastore (" +
maxStoreKeys + "
> > keys)");
> >> chkDatastore = new CHKStore();
> >> SaltedHashFreenetStore chkDataFS =
> > SaltedHashFreenetStore.construct(storeDir, "CHK-store", chkDatastore,
> >> - random, maxStoreKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook);
> >> + random, maxStoreKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook, storePreallocate);
> >> Logger.normal(this, "Initializing CHK Datacache");
> >> System.out.println("Initializing CHK Datacache (" +
maxCacheKeys + ':' +
> > maxCacheKeys + " keys)");
> >> chkDatacache = new CHKStore();
> >> SaltedHashFreenetStore chkCacheFS =
> > SaltedHashFreenetStore.construct(storeDir, "CHK-cache", chkDatacache,
> >> - random, maxCacheKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook);
> >> + random, maxCacheKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook, storePreallocate);
> >> Logger.normal(this, "Initializing pubKey
Datastore");
> >> System.out.println("Initializing pubKey
Datastore");
> >> pubKeyDatastore = new PubkeyStore();
> >> SaltedHashFreenetStore pubkeyDataFS =
> > SaltedHashFreenetStore.construct(storeDir, "PUBKEY-store",
> >> - pubKeyDatastore, random, maxStoreKeys,
bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook);
> >> + pubKeyDatastore, random, maxStoreKeys,
bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook, storePreallocate);
> >> Logger.normal(this, "Initializing pubKey
Datacache");
> >> System.out.println("Initializing pubKey Datacache
(" + maxCacheKeys + "
> > keys)");
> >> pubKeyDatacache = new PubkeyStore();
> >> SaltedHashFreenetStore pubkeyCacheFS =
> > SaltedHashFreenetStore.construct(storeDir, "PUBKEY-cache",
> >> - pubKeyDatacache, random, maxCacheKeys,
bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook);
> >> + pubKeyDatacache, random, maxCacheKeys,
bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook, storePreallocate);
> >> Logger.normal(this, "Initializing SSK Datastore");
> >> System.out.println("Initializing SSK Datastore");
> >> sskDatastore = new SSKStore(this);
> >> SaltedHashFreenetStore sskDataFS =
> > SaltedHashFreenetStore.construct(storeDir, "SSK-store", sskDatastore,
> >> - random, maxStoreKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook);
> >> + random, maxStoreKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook, storePreallocate);
> >> Logger.normal(this, "Initializing SSK Datacache");
> >> System.out.println("Initializing SSK Datacache (" +
maxCacheKeys + "
> > keys)");
> >> sskDatacache = new SSKStore(this);
> >> SaltedHashFreenetStore sskCacheFS =
> > SaltedHashFreenetStore.construct(storeDir, "SSK-cache", sskDatacache,
> >> - random, maxCacheKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook);
> >> + random, maxCacheKeys, bloomFilterSizeInM,
> > storeBloomFilterCounting, shutdownHook, storePreallocate);
> >>
> >> File migrationFile = new
File(storeDir, "migrated");
> >> if (!migrationFile.exists()) {
> >>
> >> Modified:
> > trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> >> ===================================================================
> >> ---
trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> > 2008-09-23 23:46:32 UTC (rev 22788)
> >> +++
trunk/freenet/src/freenet/store/saltedhash/SaltedHashFreenetStore.java
> > 2008-09-24 04:51:56 UTC (rev 22789)
> >> @@ -79,16 +79,18 @@
> >> private long storeSize;
> >> private int generation;
> >> private int flags;
> >> +
> >> + private boolean preallocate = true;
> >>
> >> public static SaltedHashFreenetStore construct(File baseDir, String
name,
> > StoreCallback callback, Random random,
> >> - long maxKeys, int bloomFilterSize, boolean bloomCounting,
> > SemiOrderedShutdownHook shutdownHook)
> >> + long maxKeys, int bloomFilterSize, boolean bloomCounting,
> > SemiOrderedShutdownHook shutdownHook, boolean preallocate)
> >> throws IOException {
> >> return new SaltedHashFreenetStore(baseDir, name, callback,
random,
> > maxKeys, bloomFilterSize, bloomCounting,
> >> - shutdownHook);
> >> + shutdownHook, preallocate);
> >> }
> >>
> >> private SaltedHashFreenetStore(File baseDir, String name,
StoreCallback
> > callback, Random random, long maxKeys,
> >> - int bloomFilterSize, boolean bloomCounting,
> > SemiOrderedShutdownHook shutdownHook) throws IOException {
> >> + int bloomFilterSize, boolean bloomCounting,
> > SemiOrderedShutdownHook shutdownHook, boolean preallocate) throws
IOException
> > {
> >> logMINOR = Logger.shouldLog(Logger.MINOR, this);
> >> logDEBUG = Logger.shouldLog(Logger.DEBUG, this);
> >>
> >> @@ -107,6 +109,7 @@
> >> this.random = random;
> >> storeSize = maxKeys;
> >> this.bloomFilterSize = bloomFilterSize;
> >> + this.preallocate = preallocate;
> >>
> >> lockManager = new LockManager();
> >>
> >> @@ -746,14 +749,54 @@
> >> }
> >>
> >> /**
> >> + * Set preallocate storage space
> >> + * @param preallocate
> >> + */
> >> + public void setPreallocate(boolean preallocate) {
> >> + this.preallocate = preallocate;
> >> + }
> >> +
> >> + /**
> >> * Change on disk store file size
> >> *
> >> * @param storeFileSize
> >> */
> >> private void setStoreFileSize(long storeFileSize) {
> >> try {
> >> - metaRAF.setLength(Entry.METADATA_LENGTH *
storeFileSize);
> >> - hdRAF.setLength((headerBlockLength +
dataBlockLength + hdPadding) *
> > storeFileSize);
> >> + long oldMetaLen = metaRAF.length();
> >> + long oldHdLen = hdRAF.length();
> >> +
> >> + final long newMetaLen = Entry.METADATA_LENGTH *
storeFileSize;
> >> + final long newHdLen = (headerBlockLength +
dataBlockLength + hdPadding)
> > * storeFileSize;
> >> +
> >> + if (preallocate) {
> >> + byte[] b = new byte[4096];
> >> + ByteBuffer bf = ByteBuffer.wrap(b);
> >> +
> >> + // start from next 4KB boundary => align to
x86 page size
> >> + if (oldMetaLen % 4096 != 0)
> >> + oldMetaLen += 4096 - (oldMetaLen %
4096);
> >> + if (oldHdLen % 4096 != 0)
> >> + oldHdLen += 4096 - (oldHdLen %
4096);
> >> +
> >> + // this may write excess the size, the
setLength() would fix it
> >> + while (oldMetaLen < newMetaLen) {
> >> + // never write random byte to meta
data!
> >> + // this would screw up the isFree()
function
> >> + bf.rewind();
> >> + metaFC.write(bf, oldMetaLen);
> >> + oldMetaLen += 4096;
> >> + }
> >> + while (oldHdLen < newHdLen) {
> >> + random.nextBytes(b);
> >> + bf.rewind();
> >> + hdFC.write(bf, oldHdLen);
> >> + oldHdLen += 4096;
> >> + }
> >> + }
> >> +
> >> + metaRAF.setLength(newMetaLen);
> >> + hdRAF.setLength(newHdLen);
> >> } catch (IOException e) {
> >> Logger.error(this, "error resizing store file", e);
> >> }
> >> @@ -1686,5 +1729,5 @@
> >> } finally {
> >> configLock.readLock().unlock();
> >> }
> >> - }
> >> + }
> >> }
> >>
> >> _______________________________________________
> >> cvs mailing list
> >> cvs at freenetproject.org
> >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
> >>
> >>
> >
> > _______________________________________________
> > Devl mailing list
> > Devl at freenetproject.org
> > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
> >
> _______________________________________________
> Devl mailing list
> Devl at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
>
>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 827 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20080925/1f7f7d3f/attachment.pgp>
