Author: toad
Date: 2006-02-03 01:34:42 +0000 (Fri, 03 Feb 2006)
New Revision: 7994
Added:
trunk/freenet/src/freenet/support/ReadOnlyFileSliceBucket.java
Modified:
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/support/BucketTools.java
trunk/freenet/src/freenet/support/PaddedEphemerallyEncryptedBucket.java
trunk/freenet/src/freenet/support/io/FileBucket.java
Log:
430:
Optimizations mostly related to memory usage:
- Don't keep a Mersenne Twister for every single padded encrypted bucket's
entire lifetime. They are quite big!
- Don't copy data from a file bucket to individual data buckets.
-- Reduces disk usage from 150% of file size to 50% of file size on insert.
-- Speeds up starting an insert.
-- Saves a load of memory.
Result:
~ 50MB for 3x 110MB files inserting at once (maybe a bit higher during encode
if you start all of them at once).
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2006-02-02 19:10:00 UTC (rev
7993)
+++ trunk/freenet/src/freenet/node/Version.java 2006-02-03 01:34:42 UTC (rev
7994)
@@ -20,7 +20,7 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- private static final int buildNumber = 429;
+ private static final int buildNumber = 430;
/** Oldest build of Fred we will talk to */
private static final int lastGoodBuild = 403;
Modified: trunk/freenet/src/freenet/support/BucketTools.java
===================================================================
--- trunk/freenet/src/freenet/support/BucketTools.java 2006-02-02 19:10:00 UTC
(rev 7993)
+++ trunk/freenet/src/freenet/support/BucketTools.java 2006-02-03 01:34:42 UTC
(rev 7994)
@@ -15,6 +15,8 @@
import java.util.ArrayList;
import java.util.List;
+import freenet.support.io.FileBucket;
+
/**
* Helper functions for working with Buckets.
*/
@@ -439,6 +441,9 @@
* the provided bucket, or writing to created buckets.
*/
public static Bucket[] split(Bucket origData, int splitSize,
BucketFactory bf) throws IOException {
+ if(origData instanceof FileBucket) {
+ return ((FileBucket)origData).split(splitSize);
+ }
long length = origData.size();
if(length > ((long)Integer.MAX_VALUE) * splitSize)
throw new IllegalArgumentException("Way too big!:
"+length+" for "+splitSize);
Modified:
trunk/freenet/src/freenet/support/PaddedEphemerallyEncryptedBucket.java
===================================================================
--- trunk/freenet/src/freenet/support/PaddedEphemerallyEncryptedBucket.java
2006-02-02 19:10:00 UTC (rev 7993)
+++ trunk/freenet/src/freenet/support/PaddedEphemerallyEncryptedBucket.java
2006-02-03 01:34:42 UTC (rev 7994)
@@ -20,7 +20,7 @@
private final Bucket bucket;
private final int minPaddedSize;
- private final MersenneTwister paddingSource;
+ private final RandomSource origRandom;
private final Rijndael aes;
private long dataLength;
private boolean readOnly;
@@ -35,6 +35,7 @@
* @throws UnsupportedCipherException
*/
public PaddedEphemerallyEncryptedBucket(Bucket bucket, int minSize,
RandomSource origRandom) throws UnsupportedCipherException {
+ this.origRandom = origRandom;
this.bucket = bucket;
if(bucket.size() != 0) throw new
IllegalArgumentException("Bucket must be empty");
aes = new Rijndael(256, 256);
@@ -44,7 +45,6 @@
// Might as well blank it
for(int i=0;i<key.length;i++) key[i] = 0;
this.minPaddedSize = minSize;
- paddingSource = new MersenneTwister(origRandom.nextLong());
readOnly = false;
lastOutputStream = 0;
}
@@ -107,6 +107,7 @@
return;
}
synchronized(PaddedEphemerallyEncryptedBucket.this) {
+ MersenneTwister paddingSource = new
MersenneTwister(origRandom.nextLong());
long finalLength = paddedLength();
long padding = finalLength - dataLength;
byte[] buf = new byte[4096];
Added: trunk/freenet/src/freenet/support/ReadOnlyFileSliceBucket.java
===================================================================
--- trunk/freenet/src/freenet/support/ReadOnlyFileSliceBucket.java
2006-02-02 19:10:00 UTC (rev 7993)
+++ trunk/freenet/src/freenet/support/ReadOnlyFileSliceBucket.java
2006-02-03 01:34:42 UTC (rev 7994)
@@ -0,0 +1,103 @@
+package freenet.support;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+
+/**
+ * FIXME: implement a hash verifying version of this.
+ */
+public class ReadOnlyFileSliceBucket implements Bucket {
+
+ private final File file;
+ private final long startAt;
+ private final long length;
+
+ public ReadOnlyFileSliceBucket(File f, long startAt, long length) {
+ this.file = f;
+ this.startAt = startAt;
+ this.length = length;
+ }
+
+ public OutputStream getOutputStream() throws IOException {
+ throw new IOException("Bucket is read-only");
+ }
+
+ public InputStream getInputStream() throws IOException {
+ return new MyInputStream();
+ }
+
+ public String getName() {
+ return "ROFS:"+file.getAbsolutePath()+":"+startAt+":"+length;
+ }
+
+ public long size() {
+ return length;
+ }
+
+ public boolean isReadOnly() {
+ return true;
+ }
+
+ public void setReadOnly() {
+ // Do nothing
+ }
+
+ class MyInputStream extends InputStream {
+
+ private RandomAccessFile f;
+ private long ptr; // relative to startAt
+
+ MyInputStream() throws IOException {
+ try {
+ this.f = new RandomAccessFile(file,"r");
+ f.seek(startAt);
+ if(f.length() < (startAt+length))
+ throw new
ReadOnlyFileSliceBucketException("File truncated? Length "+f.length()+" but
start at "+startAt+" for "+length+" bytes");
+ ptr = 0;
+ } catch (FileNotFoundException e) {
+ throw new ReadOnlyFileSliceBucketException(e);
+ }
+ }
+
+ public int read() throws IOException {
+ if(ptr > length)
+ throw new EOFException();
+ int x = f.read();
+ ptr++;
+ return x;
+ }
+
+ public int read(byte[] buf, int offset, int len) throws
IOException {
+ if(ptr > length)
+ throw new EOFException();
+ len = (int) Math.min(len, length - ptr);
+ int x = f.read(buf, offset, len);
+ ptr += x;
+ return x;
+ }
+
+ public int read(byte[] buf) throws IOException {
+ return read(buf, 0, buf.length);
+ }
+
+ }
+
+ public class ReadOnlyFileSliceBucketException extends IOException {
+
+ public ReadOnlyFileSliceBucketException(FileNotFoundException
e) {
+ super("File not found: "+e.getMessage());
+ initCause(e);
+ }
+
+ public ReadOnlyFileSliceBucketException(String string) {
+ super(string);
+ }
+
+ }
+
+}
Modified: trunk/freenet/src/freenet/support/io/FileBucket.java
===================================================================
--- trunk/freenet/src/freenet/support/io/FileBucket.java 2006-02-02
19:10:00 UTC (rev 7993)
+++ trunk/freenet/src/freenet/support/io/FileBucket.java 2006-02-03
01:34:42 UTC (rev 7994)
@@ -11,6 +11,7 @@
import freenet.crypt.RandomSource;
import freenet.support.Bucket;
import freenet.support.Logger;
+import freenet.support.ReadOnlyFileSliceBucket;
/**
* A file Bucket is an implementation of Bucket that writes to a file.
@@ -299,4 +300,19 @@
public void dontDeleteOnFinalize() {
deleteOnFinalize = false;
}
+
+ public Bucket[] split(int splitSize) {
+ if(length > ((long)Integer.MAX_VALUE) * splitSize)
+ throw new IllegalArgumentException("Way too big!:
"+length+" for "+splitSize);
+ int bucketCount = (int) (length / splitSize);
+ if(length % splitSize > 0) bucketCount++;
+ Bucket[] buckets = new Bucket[bucketCount];
+ for(int i=0;i<buckets.length;i++) {
+ long startAt = i * splitSize;
+ long endAt = Math.min((i+1) * splitSize, length);
+ long len = endAt - startAt;
+ buckets[i] = new ReadOnlyFileSliceBucket(file, startAt,
len);
+ }
+ return buckets;
+ }
}
