Author: toad
Date: 2009-02-17 15:08:07 +0000 (Tue, 17 Feb 2009)
New Revision: 25666
Modified:
branches/db4o/freenet/src/freenet/client/async/ClientContext.java
branches/db4o/freenet/src/freenet/support/api/BucketFactory.java
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
branches/db4o/freenet/src/freenet/support/io/SegmentedBucketChainBucketKillJob.java
Log:
Run free jobs immediately before extending the file. We are still getting huge
persistent blobs...
Modified: branches/db4o/freenet/src/freenet/client/async/ClientContext.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/ClientContext.java
2009-02-17 13:09:56 UTC (rev 25665)
+++ branches/db4o/freenet/src/freenet/client/async/ClientContext.java
2009-02-17 15:08:07 UTC (rev 25666)
@@ -22,6 +22,7 @@
import freenet.support.compress.RealCompressor;
import freenet.support.io.FilenameGenerator;
import freenet.support.io.NativeThread;
+import freenet.support.io.PersistentTempBucketFactory;
/**
* Object passed in to client-layer operations, containing references to
essential but transient objects
@@ -41,7 +42,7 @@
public transient final BackgroundBlockEncoder backgroundBlockEncoder;
public transient final RandomSource random;
public transient final ArchiveManager archiveManager;
- public transient final BucketFactory persistentBucketFactory;
+ public transient final PersistentTempBucketFactory
persistentBucketFactory;
public transient final BucketFactory tempBucketFactory;
public transient final HealingQueue healingQueue;
public transient final USKManager uskManager;
@@ -54,7 +55,7 @@
public ClientContext(NodeClientCore core, FECQueue fecQueue, Executor
mainExecutor,
BackgroundBlockEncoder blockEncoder, ArchiveManager
archiveManager,
- BucketFactory ptbf, BucketFactory tbf, HealingQueue hq,
+ PersistentTempBucketFactory ptbf, BucketFactory tbf,
HealingQueue hq,
USKManager uskManager, RandomSource strongRandom,
Random fastWeakRandom, Ticker ticker,
FilenameGenerator fg, FilenameGenerator persistentFG,
RealCompressor rc) {
Modified: branches/db4o/freenet/src/freenet/support/api/BucketFactory.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/api/BucketFactory.java
2009-02-17 13:09:56 UTC (rev 25665)
+++ branches/db4o/freenet/src/freenet/support/api/BucketFactory.java
2009-02-17 15:08:07 UTC (rev 25666)
@@ -5,7 +5,9 @@
import java.io.IOException;
+import freenet.support.io.SegmentedBucketChainBucketKillJob;
+
public interface BucketFactory {
/**
* Create a bucket.
Modified:
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
===================================================================
---
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
2009-02-17 13:09:56 UTC (rev 25665)
+++
branches/db4o/freenet/src/freenet/support/io/PersistentBlobTempBucketFactory.java
2009-02-17 15:08:07 UTC (rev 25666)
@@ -9,6 +9,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -41,6 +42,7 @@
public final long blockSize;
private File storageFile;
private transient RandomAccessFile raf;
+ private transient LinkedList<DBJob> freeJobs;
/** We use NIO for the equivalent of pwrite/pread. This is parallelized
on unix
* but sadly not on Windows. */
transient FileChannel channel;
@@ -93,7 +95,69 @@
shadows = new TreeMap<Long,PersistentBlobTempBucket>();
jobRunner = jobRunner2;
weakRandomSource = fastWeakRandom;
+ freeJobs = new LinkedList<DBJob>();
this.ticker = ticker;
+
+ // Diagnostics
+
+ long size;
+ try {
+ size = channel.size();
+ } catch (IOException e1) {
+ Logger.error(this, "Unable to find size of temp blob
storage file: "+e1, e1);
+ return;
+ }
+ size -= size % blockSize;
+ long blocks = size / blockSize;
+ long ptr = blocks - 1;
+
+ long used = 0;
+ long rangeStart = Long.MIN_VALUE;
+ PersistentBlobTempBucketTag firstInRange = null;
+ for(long l = 0; l < ptr; l++) {
+ synchronized(this) {
+ if(freeSlots.containsKey(l)) continue;
+ if(notCommittedBlobs.containsKey(l)) continue;
+ if(almostFreeSlots.containsKey(l)) continue;
+ }
+ Query query = container.query();
+ query.constrain(PersistentBlobTempBucketTag.class);
+ query.descend("index").constrain(l);
+ ObjectSet<PersistentBlobTempBucketTag> tags =
query.execute();
+ if(tags.hasNext()) {
+ PersistentBlobTempBucketTag tag = tags.next();
+ if(!tag.isFree)
+ used++;
+ if(tag.bucket == null && !tag.isFree)
+ Logger.error(this, "No bucket but
flagged as not free: index "+l+" "+tag.bucket);
+ if(tag.bucket != null && tag.isFree)
+ Logger.error(this, "Has bucket but
flagged as free: index "+l+" "+tag.bucket);
+ if(!tag.isFree) {
+ if(rangeStart == Long.MIN_VALUE) {
+ rangeStart = l;
+ firstInRange = tag;
+ }
+ } else {
+ if(rangeStart != Long.MIN_VALUE) {
+ System.out.println("Range:
"+rangeStart+" to "+(l-1)+" first is "+firstInRange);
+ rangeStart = Long.MIN_VALUE;
+ firstInRange = null;
+ }
+ }
+ continue;
+ }
+ Logger.error(this, "FOUND EMPTY SLOT: "+l+" when
scanning the blob file because tags in database < length of file");
+ PersistentBlobTempBucketTag tag = new
PersistentBlobTempBucketTag(PersistentBlobTempBucketFactory.this, l);
+ container.store(tag);
+ synchronized(this) {
+ freeSlots.put(ptr, tag);
+ }
+ }
+ if(rangeStart != Long.MIN_VALUE) {
+ System.out.println("Range: "+rangeStart+" to "+(ptr-1));
+ }
+ System.err.println("Persistent blobs: Blocks: "+blocks+" used
"+used);
+
}
public String getName() {
@@ -105,6 +169,7 @@
private final DBJob slotFinder = new DBJob() {
public void run(ObjectContainer container, ClientContext
context) {
+ while(true) {
boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
synchronized(PersistentBlobTempBucketFactory.this) {
if(freeSlots.size() > MAX_FREE) return;
@@ -197,6 +262,18 @@
}
}
+ DBJob freeJob = null;
+ synchronized(this) {
+ if(!freeJobs.isEmpty())
+ freeJob = freeJobs.removeFirst();
+ }
+ if(freeJob != null) {
+ container.activate(freeJob, 1);
+ System.err.println("Freeing some space by
running "+freeJob);
+ freeJob.run(container, context);
+ continue;
+ }
+
// Lets extend the file.
// FIXME if physical security is LOW, just set the
length, possibly
// padding will nonrandom nulls on unix.
@@ -242,7 +319,9 @@
freeSlots.put(ptr, tag);
}
}
+ return;
}
+ }
};
@@ -509,10 +588,11 @@
}
public synchronized void postCommit() {
- int sz = freeSlots.size() + almostFreeSlots.size();
+ int freeNow = freeSlots.size();
+ int sz = freeNow + almostFreeSlots.size();
if(sz > MAX_FREE) {
Iterator<Map.Entry<Long,PersistentBlobTempBucketTag>>
it = almostFreeSlots.entrySet().iterator();
- for(int i=sz;i<MAX_FREE && it.hasNext();i++) {
+ for(int i=freeNow;i<MAX_FREE && it.hasNext();i++) {
Map.Entry<Long,PersistentBlobTempBucketTag>
entry = it.next();
freeSlots.put(entry.getKey(), entry.getValue());
}
@@ -542,4 +622,10 @@
}
}
+ public void addBlobFreeCallback(DBJob job) {
+ synchronized(this) {
+ freeJobs.add(job);
+ }
+ }
+
}
Modified:
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
===================================================================
---
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
2009-02-17 13:09:56 UTC (rev 25665)
+++
branches/db4o/freenet/src/freenet/support/io/PersistentTempBucketFactory.java
2009-02-17 15:08:07 UTC (rev 25666)
@@ -8,12 +8,14 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.Random;
import com.db4o.ObjectContainer;
import com.db4o.ObjectSet;
import com.db4o.query.Predicate;
+import freenet.client.async.DBJob;
import freenet.client.async.DBJobRunner;
import freenet.crypt.RandomSource;
import freenet.keys.CHKBlock;
@@ -244,4 +246,8 @@
db.commit();
}
}
+
+ public void addBlobFreeCallback(DBJob job) {
+ blobFactory.addBlobFreeCallback(job);
+ }
}
Modified:
branches/db4o/freenet/src/freenet/support/io/SegmentedBucketChainBucketKillJob.java
===================================================================
---
branches/db4o/freenet/src/freenet/support/io/SegmentedBucketChainBucketKillJob.java
2009-02-17 13:09:56 UTC (rev 25665)
+++
branches/db4o/freenet/src/freenet/support/io/SegmentedBucketChainBucketKillJob.java
2009-02-17 15:08:07 UTC (rev 25666)
@@ -25,6 +25,7 @@
// More work needs to be done.
// We will have already been removed, so re-add, in
case we crash soon.
scheduleRestart(container, context);
+
context.persistentBucketFactory.addBlobFreeCallback(this);
// But try to sort it out now ...
context.jobRunner.queue(this,
NativeThread.NORM_PRIORITY, true);
} else {
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
