Author: nextgens
Date: 2008-08-13 13:21:16 +0000 (Wed, 13 Aug 2008)
New Revision: 21801
Removed:
trunk/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucketFactory.java
Modified:
trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
trunk/freenet/src/freenet/node/NodeClientCore.java
trunk/freenet/src/freenet/support/io/TempBucketFactory.java
Log:
Untested patch introducing RAMBuckets. All small (below 32KiB buckets) are
allocated in ram as ArrayBuckets until we reach a max capacity (10MiB by
default).
Both parameters are config options. It should speedup the content-filter and
more generally speaking fproxy significantly.
Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties
===================================================================
--- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-08-13
12:29:17 UTC (rev 21800)
+++ trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2008-08-13
13:21:16 UTC (rev 21801)
@@ -734,6 +734,10 @@
NodeClientCore.movingTempDirOnTheFlyNotSupported=Moving temp directory on the
fly not supported at present
NodeClientCore.persistentTempDir=Persistent temp files directory
NodeClientCore.persistentTempDirLong=Name of directory to put persistent temp
files in
+NodeClientCore.maxRAMBucketSize=Maximum size of a RAMBucket
+NodeClientCore.maxRAMBucketSizeLong=Maximum size of a RAMBucket (bigger
buckets will be kept as files on the disk)
+NodeClientCore.ramBucketPoolSize=Amount of RAM to dedicate to temporary buckets
+NodeClientCore.ramBucketPoolSizeLong=Amount of RAM to dedicate to temporary
buckets. The tradeoff is more memory usage against more IOs.
NodeClientCore.startingUpTitle=Freenet is starting up
NodeClientCore.startingUp=Please allow Freenet a few moments to complete the
startup process, in the meantime some things may not work and the node may be
slower than usual.
NodeClientCore.startingUpShort=Freenet is starting up, some things may not
work and the node may be slow.
Modified: trunk/freenet/src/freenet/node/NodeClientCore.java
===================================================================
--- trunk/freenet/src/freenet/node/NodeClientCore.java 2008-08-13 12:29:17 UTC
(rev 21800)
+++ trunk/freenet/src/freenet/node/NodeClientCore.java 2008-08-13 13:21:16 UTC
(rev 21801)
@@ -53,13 +53,12 @@
import freenet.support.SimpleFieldSet;
import freenet.support.api.BooleanCallback;
import freenet.support.api.IntCallback;
-import freenet.support.api.BucketFactory;
+import freenet.support.api.LongCallback;
import freenet.support.api.StringArrCallback;
import freenet.support.api.StringCallback;
import freenet.support.io.FileUtil;
import freenet.support.io.FilenameGenerator;
import freenet.support.io.NativeThread;
-import freenet.support.io.PaddedEphemerallyEncryptedBucketFactory;
import freenet.support.io.PersistentTempBucketFactory;
import freenet.support.io.TempBucketFactory;
@@ -82,7 +81,7 @@
private File[] uploadAllowedDirs;
private boolean uploadAllowedEverywhere;
final FilenameGenerator tempFilenameGenerator;
- public final PaddedEphemerallyEncryptedBucketFactory tempBucketFactory;
+ public final TempBucketFactory tempBucketFactory;
public final PersistentTempBucketFactory persistentTempBucketFactory;
public final Node node;
final NodeStats nodeStats;
@@ -204,6 +203,29 @@
throw new
NodeInitException(NodeInitException.EXIT_BAD_TEMP_DIR, msg);
}
+ nodeConfig.register("maxRAMBucketSize", "32KiB", sortOrder++,
true, false, "NodeClientCore.maxRAMBucketSize",
"NodeClientCore.maxRAMBucketSizeLong", new LongCallback() {
+
+ public long get() {
+ return (tempBucketFactory == null ? 0 :
tempBucketFactory.getMaxRAMBucketSize());
+ }
+
+ public void set(long val) throws
InvalidConfigValueException {
+ if((val == get()) || (tempBucketFactory ==
null)) return;
+ tempBucketFactory.setMaxRAMBucketSize(val);
+ }
+ });
+ nodeConfig.register("RAMBucketPoolSize", "10MiB", sortOrder++,
true, false, "NodeClientCore.ramBucketPoolSize",
"NodeClientCore.ramBucketPoolSizeLong", new LongCallback() {
+
+ public long get() {
+ return (tempBucketFactory == null ? 0 :
tempBucketFactory.getMaxRamUsed());
+ }
+
+ public void set(long val) throws
InvalidConfigValueException {
+ if((val == get()) || (tempBucketFactory ==
null)) return;
+ tempBucketFactory.setMaxRamUsed(val);
+ }
+ });
+
nodeConfig.register("encryptTempBuckets", true, sortOrder++,
true, false, "NodeClientCore.encryptTempBuckets",
"NodeClientCore.encryptTempBucketsLong", new BooleanCallback() {
public boolean get() {
@@ -215,8 +237,7 @@
tempBucketFactory.setEncryption(val);
}
});
- BucketFactory _tempBucketFactory = new
TempBucketFactory(tempFilenameGenerator);
- tempBucketFactory = new
PaddedEphemerallyEncryptedBucketFactory(_tempBucketFactory, random,
node.fastWeakRandom, 1024, nodeConfig.getBoolean("encryptTempBuckets"));
+ tempBucketFactory = new
TempBucketFactory(tempFilenameGenerator,
nodeConfig.getLong("maxRAMBucketSize"),
nodeConfig.getLong("RAMBucketPoolSize"), random, node.fastWeakRandom,
nodeConfig.getBoolean("encryptTempBuckets"));
// Downloads directory
Deleted:
trunk/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucketFactory.java
===================================================================
---
trunk/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucketFactory.java
2008-08-13 12:29:17 UTC (rev 21800)
+++
trunk/freenet/src/freenet/support/io/PaddedEphemerallyEncryptedBucketFactory.java
2008-08-13 13:21:16 UTC (rev 21801)
@@ -1,45 +0,0 @@
-package freenet.support.io;
-
-import freenet.crypt.RandomSource;
-import java.io.IOException;
-
-import freenet.support.api.Bucket;
-import freenet.support.api.BucketFactory;
-import java.util.Random;
-
-/**
- * Factory wrapper for PaddedEphemerallyEncryptedBucket's, which are themselves
- * wrappers.
- */
-public class PaddedEphemerallyEncryptedBucketFactory implements BucketFactory {
-
- final BucketFactory baseFactory;
- final RandomSource strongPRNG;
- final Random weakPRNG;
- final int minSize;
- private volatile boolean reallyEncrypt;
-
- public PaddedEphemerallyEncryptedBucketFactory(BucketFactory factory,
RandomSource strongPRNG, Random weakPRNG, int minSize, boolean reallyEncrypt) {
- baseFactory = factory;
- this.minSize = minSize;
- this.strongPRNG = strongPRNG;
- this.weakPRNG = weakPRNG;
- this.reallyEncrypt = reallyEncrypt;
- }
-
- public Bucket makeBucket(long size) throws IOException {
- Bucket realBucket = baseFactory.makeBucket(size);
- if(!reallyEncrypt)
- return realBucket;
- else
- return new PaddedEphemerallyEncryptedBucket(realBucket,
minSize, strongPRNG, weakPRNG);
- }
-
- public void setEncryption(boolean value) {
- reallyEncrypt = value;
- }
-
- public boolean isEncrypting() {
- return reallyEncrypt;
- }
-}
Modified: trunk/freenet/src/freenet/support/io/TempBucketFactory.java
===================================================================
--- trunk/freenet/src/freenet/support/io/TempBucketFactory.java 2008-08-13
12:29:17 UTC (rev 21800)
+++ trunk/freenet/src/freenet/support/io/TempBucketFactory.java 2008-08-13
13:21:16 UTC (rev 21801)
@@ -1,5 +1,6 @@
package freenet.support.io;
+import freenet.crypt.RandomSource;
import java.io.IOException;
import freenet.support.api.Bucket;
@@ -10,6 +11,7 @@
* distributed under the GNU Public Licence (GPL) version 2. See
* http://www.gnu.org/ for further details of the GPL.
*/
+import java.util.Random;
/**
* Temporary Bucket Factory
@@ -18,15 +20,45 @@
*/
public class TempBucketFactory implements BucketFactory {
+ private class RAMBucket extends ArrayBucket {
+ private final long size;
+
+ public RAMBucket(long size) {
+ super("RAMBucket");
+ this.size = size;
+ }
+
+ @Override
+ public void free() {
+ super.free();
+ _hasFreed(size);
+ }
+ }
+
private final FilenameGenerator filenameGenerator;
+ private final ArrayBucketFactory arrayBucketFactory;
+ private long bytesInUse = 0;
public final static long defaultIncrement = 4096;
public final static float DEFAULT_FACTOR = 1.25F;
+
+ public long maxRAMBucketSize;
+ public long maxRamUsed;
+ final RandomSource strongPRNG;
+ final Random weakPRNG;
+ private volatile boolean reallyEncrypt;
+
// Storage accounting disabled by default.
- public TempBucketFactory(FilenameGenerator filenameGenerator) {
+ public TempBucketFactory(FilenameGenerator filenameGenerator, long
maxBucketSizeKeptInRam, long maxRamUsed, RandomSource strongPRNG, Random
weakPRNG, boolean reallyEncrypt) {
this.filenameGenerator = filenameGenerator;
+ this.arrayBucketFactory = new ArrayBucketFactory();
+ this.maxRamUsed = maxRamUsed;
+ this.maxRAMBucketSize = maxBucketSizeKeptInRam;
+ this.strongPRNG = strongPRNG;
+ this.weakPRNG = weakPRNG;
+ this.reallyEncrypt = reallyEncrypt;
}
public Bucket makeBucket(long size) throws IOException {
@@ -36,6 +68,34 @@
public Bucket makeBucket(long size, float factor) throws IOException {
return makeBucket(size, factor, defaultIncrement);
}
+
+ protected synchronized void _hasFreed(long size) {
+ bytesInUse -= size;
+ }
+
+ public synchronized void setMaxRamUsed(long size) {
+ maxRamUsed = size;
+ }
+
+ public synchronized long getMaxRamUsed() {
+ return maxRamUsed;
+ }
+
+ public synchronized void setMaxRAMBucketSize(long size) {
+ maxRAMBucketSize = size;
+ }
+
+ public synchronized long getMaxRAMBucketSize() {
+ return maxRAMBucketSize;
+ }
+
+ public void setEncryption(boolean value) {
+ reallyEncrypt = value;
+ }
+
+ public boolean isEncrypting() {
+ return reallyEncrypt;
+ }
/**
* Create a temp bucket
@@ -49,11 +109,19 @@
* If it is not possible to create a temp bucket due to
an
* I/O error
*/
- public Bucket makeBucket(long size, float factor, long increment)
- throws IOException {
- long id = filenameGenerator.makeRandomFilename();
-
- return new TempFileBucket(id, filenameGenerator);
+ public Bucket makeBucket(long size, float factor, long increment)
throws IOException {
+ Bucket realBucket = null;
+ boolean isARAMBucket = false;
+
+ synchronized(this) {
+ if((size > 0) && (size < maxRAMBucketSize) &&
(bytesInUse < maxRamUsed)) {
+ bytesInUse += size;
+ isARAMBucket = true;
+ }
+ }
+
+ realBucket = (isARAMBucket ? new RAMBucket(size) : new
TempFileBucket(filenameGenerator.makeRandomFilename(), filenameGenerator));
+
+ return (!reallyEncrypt ? realBucket : new
PaddedEphemerallyEncryptedBucket(realBucket, 1024, strongPRNG, weakPRNG));
}
-
}
