Author: toad
Date: 2009-01-24 18:27:06 +0000 (Sat, 24 Jan 2009)
New Revision: 25286
Modified:
branches/db4o/freenet/src/freenet/node/NodeClientCore.java
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
Log:
Shrink the persistent blob file when possible.
Modified: branches/db4o/freenet/src/freenet/node/NodeClientCore.java
===================================================================
--- branches/db4o/freenet/src/freenet/node/NodeClientCore.java 2009-01-24
18:26:26 UTC (rev 25285)
+++ branches/db4o/freenet/src/freenet/node/NodeClientCore.java 2009-01-24
18:27:06 UTC (rev 25286)
@@ -251,7 +251,7 @@
try {
File dir = new
File(nodeConfig.getString("persistentTempDir"));
String prefix = "freenet-temp-";
- persistentTempBucketFactory =
PersistentTempBucketFactory.load(dir, prefix, random, node.fastWeakRandom,
container, node.nodeDBHandle,
nodeConfig.getBoolean("encryptPersistentTempBuckets"), this);
+ persistentTempBucketFactory =
PersistentTempBucketFactory.load(dir, prefix, random, node.fastWeakRandom,
container, node.nodeDBHandle,
nodeConfig.getBoolean("encryptPersistentTempBuckets"), this, node.getTicker());
persistentTempBucketFactory.init(dir, prefix, random,
node.fastWeakRandom);
persistentFilenameGenerator =
persistentTempBucketFactory.fg;
} catch(IOException e2) {
@@ -1381,7 +1381,7 @@
if(bucket.toRemove())
bucket.realRemoveFrom(node.db);
} catch (Throwable t) {
- Logger.error(this, "Caught
"+t+" freeing bucket "+bucket+" after transaction commit");
+ Logger.error(this, "Caught
"+t+" freeing bucket "+bucket+" after transaction commit", t);
}
}
} catch (Throwable t) {
Modified:
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
===================================================================
---
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
2009-01-24 18:26:26 UTC (rev 25285)
+++
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
2009-01-24 18:27:06 UTC (rev 25286)
@@ -19,6 +19,7 @@
import freenet.client.async.ClientContext;
import freenet.client.async.DBJob;
import freenet.client.async.DBJobRunner;
+import freenet.node.Ticker;
import freenet.support.Logger;
/**
@@ -42,7 +43,7 @@
transient FileChannel channel;
/** Blobs in memory only: in the database there will still be a "free"
tag */
- private transient Map<Long,PersistentBlobTempBucket> notCommittedBlobs;
+ private transient TreeMap<Long,PersistentBlobTempBucket>
notCommittedBlobs;
/** Non-exhaustive list of free slots. If we run out we need to query
for
* more. */
@@ -52,6 +53,8 @@
private transient Random weakRandomSource;
+ private transient Ticker ticker;
+
private final long nodeDBHandle;
public PersistentBlobTempBucketFactory(long blockSize2, long
nodeDBHandle2, File storageFile2) {
@@ -60,7 +63,7 @@
storageFile = storageFile2;
}
- void onInit(ObjectContainer container, DBJobRunner jobRunner2, Random
fastWeakRandom, File storageFile2, long blockSize2) throws IOException {
+ void onInit(ObjectContainer container, DBJobRunner jobRunner2, Random
fastWeakRandom, File storageFile2, long blockSize2, Ticker ticker) throws
IOException {
container.activate(storageFile, 100);
if(storageFile2.getPath().equals(storageFile.getPath())) {
if(blockSize != blockSize2)
@@ -73,10 +76,11 @@
}
raf = new RandomAccessFile(storageFile, "rw");
channel = raf.getChannel();
- notCommittedBlobs = new
HashMap<Long,PersistentBlobTempBucket>();
+ notCommittedBlobs = new
TreeMap<Long,PersistentBlobTempBucket>();
freeSlots = new TreeMap<Long,PersistentBlobTempBucketTag>();
jobRunner = jobRunner2;
weakRandomSource = fastWeakRandom;
+ this.ticker = ticker;
}
public String getName() {
@@ -292,13 +296,18 @@
}
}
+ private long lastCheckedEnd = -1;
+
public synchronized void remove(PersistentBlobTempBucket bucket,
ObjectContainer container) {
if(Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "Removing bucket "+bucket+" for slot
"+bucket.index+" from database", new Exception("debug"));
long index = bucket.index;
PersistentBlobTempBucketTag tag = bucket.tag;
container.activate(tag, 1);
- if(!bucket.persisted()) return;
+ if(!bucket.persisted()) {
+ maybeShrink(container);
+ return;
+ }
if(!bucket.freed()) {
Logger.error(this, "Removing bucket "+bucket+" for slot
"+bucket.index+" but not freed!", new Exception("debug"));
notCommittedBlobs.put(index, bucket);
@@ -310,8 +319,117 @@
container.store(tag);
container.delete(bucket);
bucket.onRemove();
+
+ maybeShrink(container);
}
+
+ void maybeShrink(ObjectContainer container) {
+
+ boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
+ if(logMINOR) Logger.minor(this, "maybeShrink()");
+ long now = System.currentTimeMillis();
+
+ long newBlocks;
+
+ synchronized(this) {
+
+ if(now - lastCheckedEnd > 60*1000) {
+ if(logMINOR) Logger.minor(this, "maybeShrink() inner");
+ // Check whether there is a big white space at the end
of the file.
+ long size;
+ try {
+ size = channel.size();
+ } catch (IOException e1) {
+ Logger.error(this, "Unable to find size of temp
blob storage file: "+e1, e1);
+ return;
+ }
+ size -= size % blockSize;
+ long blocks = (size / blockSize) - 1;
+ if(blocks <= 32) {
+ if(logMINOR) Logger.minor(this, "Not shrinking,
blob file not larger than a megabyte");
+ lastCheckedEnd = now;
+ queueMaybeShrink();
+ return;
+ }
+ long lastNotCommitted = notCommittedBlobs.isEmpty() ? 0
: notCommittedBlobs.lastKey();
+ double full = (double)lastNotCommitted / (double)blocks;
+ if(full > 0.8) {
+ if(logMINOR) Logger.minor(this, "Not shrinking,
last not committed block is at "+full*100+"% ("+lastNotCommitted+" of
"+blocks+")");
+ lastCheckedEnd = now;
+ queueMaybeShrink();
+ return;
+ }
+ Query query = container.query();
+ query.constrain(PersistentBlobTempBucketTag.class);
+ query.descend("isFree").constrain(false);
+ query.descend("index").orderDescending();
+ ObjectSet<PersistentBlobTempBucketTag> tags =
query.execute();
+ long lastCommitted;
+ if(tags.isEmpty()) {
+ // No used slots at all?!
+ // There may be some not committed though
+ Logger.normal(this, "No used slots in
persistent temp file (but last not committed = "+lastNotCommitted+")");
+ lastCommitted = 0;
+ } else {
+ lastCommitted = tags.next().index;
+ if(logMINOR) Logger.minor(this, "Last committed
slot is "+lastCommitted+" last not committed is "+lastNotCommitted);
+ }
+ full = (double) lastCommitted / (double) blocks;
+ if(full > 0.8) {
+ if(logMINOR) Logger.minor(this, "Not shrinking,
last committed block is at "+full*100+"%");
+ lastCheckedEnd = now;
+ queueMaybeShrink();
+ return;
+ }
+ long lastBlock = Math.max(lastCommitted,
lastNotCommitted);
+ // Must be 10% free at end
+ newBlocks = (long) ((lastBlock + 32) * (1.0 / 1.1));
+ newBlocks = Math.max(newBlocks, 32);
+ System.err.println("Shrinking blob file from "+blocks+"
to "+newBlocks);
+ for(long l = newBlocks; l <= blocks; l++) {
+ freeSlots.remove(l);
+ }
+ for(Long l : freeSlots.keySet()) {
+ if(l > newBlocks) {
+ Logger.error(this, "Removing free slot
"+l+" over the current block limit");
+ }
+ }
+ lastCheckedEnd = now;
+ queueMaybeShrink();
+ } else return;
+ }
+ try {
+ channel.truncate(newBlocks * blockSize);
+ } catch (IOException e) {
+ System.err.println("Shrinking blob file failed!");
+ System.err.println(e);
+ e.printStackTrace();
+ Logger.error(this, "Shrinking blob file failed!: "+e,
e);
+ }
+ Query query = container.query();
+ query.constrain(PersistentBlobTempBucketTag.class);
+ query.descend("index").constrain(newBlocks).greater();
+ ObjectSet<PersistentBlobTempBucketTag> tags = query.execute();
+ while(tags.hasNext()) container.delete(tags.next());
+
+ }
+ private void queueMaybeShrink() {
+ ticker.queueTimedJob(new Runnable() {
+
+ public void run() {
+ jobRunner.queue(new DBJob() {
+
+ public void run(ObjectContainer
container, ClientContext context) {
+ maybeShrink(container);
+ }
+
+ }, NativeThread.NORM_PRIORITY-1, true);
+ }
+
+ }, 61*1000);
+ }
+
public void store(PersistentBlobTempBucket bucket, ObjectContainer
container) {
if(Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "Storing bucket "+bucket+" for slot
"+bucket.index+" to database");
Modified:
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
===================================================================
---
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
2009-01-24 18:26:26 UTC (rev 25285)
+++
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
2009-01-24 18:27:06 UTC (rev 25286)
@@ -17,6 +17,7 @@
import freenet.client.async.DBJobRunner;
import freenet.crypt.RandomSource;
import freenet.keys.CHKBlock;
+import freenet.node.Ticker;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
@@ -175,7 +176,7 @@
return encrypt;
}
- public static PersistentTempBucketFactory load(File dir, String prefix,
RandomSource random, Random fastWeakRandom, ObjectContainer container, final
long nodeDBHandle, boolean encrypt, DBJobRunner jobRunner) throws IOException {
+ public static PersistentTempBucketFactory load(File dir, String prefix,
RandomSource random, Random fastWeakRandom, ObjectContainer container, final
long nodeDBHandle, boolean encrypt, DBJobRunner jobRunner, Ticker ticker)
throws IOException {
ObjectSet<PersistentTempBucketFactory> results =
container.query(new Predicate<PersistentTempBucketFactory>() {
public boolean match(PersistentTempBucketFactory
factory) {
if(factory.nodeDBHandle == nodeDBHandle) return
true;
@@ -187,12 +188,12 @@
container.activate(factory, 5);
factory.init(dir, prefix, random, fastWeakRandom);
factory.setEncryption(encrypt);
- factory.blobFactory.onInit(container, jobRunner,
fastWeakRandom, new File(dir, "persistent-blob.tmp"), BLOB_SIZE);
+ factory.blobFactory.onInit(container, jobRunner,
fastWeakRandom, new File(dir, "persistent-blob.tmp"), BLOB_SIZE, ticker);
return factory;
} else {
PersistentTempBucketFactory factory =
new PersistentTempBucketFactory(dir, prefix,
random, fastWeakRandom, encrypt, nodeDBHandle);
- factory.blobFactory.onInit(container, jobRunner,
fastWeakRandom, new File(dir, "persistent-blob.tmp"), BLOB_SIZE);
+ factory.blobFactory.onInit(container, jobRunner,
fastWeakRandom, new File(dir, "persistent-blob.tmp"), BLOB_SIZE, ticker);
return factory;
}
}
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
