Author: toad
Date: 2007-03-15 20:25:50 +0000 (Thu, 15 Mar 2007)
New Revision: 12140
Modified:
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
Log:
Enable online store shrinking when it is less than a 10% reduction.
Add node.storeForceBigShrinks option (force online store shrink even if it is
over 10% of store size).
Some method refactoring in store impl.
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2007-03-15 20:07:28 UTC (rev
12139)
+++ trunk/freenet/src/freenet/node/Node.java 2007-03-15 20:25:50 UTC (rev
12140)
@@ -276,6 +276,10 @@
private long maxStoreKeys;
/** The maximum size of the datastore. Kept to avoid rounding turning
5G into 5368698672 */
private long maxTotalDatastoreSize;
+ /** If true, store shrinks occur immediately even if they are over 10%
of the store size. If false,
+ * we just set the storeSize and do an offline shrink on the next
startup. Online shrinks do not
+ * preserve the most recently used data so are not recommended. */
+ private boolean storeForceBigShrinks;
private StatsConfig statsConf;
/* These are private because must be protected by synchronized(this) */
@@ -1153,8 +1157,25 @@
new NodeNameCallback(this));
myName = nodeConfig.getString("name");
+ // Datastore
- // Datastore
+ nodeConfig.register("storeForceBigShrinks", false, sortOrder++,
true, false, "Do large store shrinks immediately", "Whether to do large store
shrinks (over 10%) immediately (rather than waiting for the next node restart).
Online shrinks do not preserve the most recently used data, so this is not
recommended; use it only if you must have an immediate result.",
+ new BooleanCallback() {
+
+ public boolean get() {
+ synchronized(Node.this) {
+ return
storeForceBigShrinks;
+ }
+ }
+
+ public void set(boolean val) throws
InvalidConfigValueException {
+ synchronized(Node.this) {
+ storeForceBigShrinks =
val;
+ }
+ }
+
+ });
+
nodeConfig.register("storeSize", "1G", sortOrder++, false,
true, "Store size in bytes", "Store size in bytes",
new LongCallback() {
Modified: trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java
===================================================================
--- trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2007-03-15
20:07:28 UTC (rev 12139)
+++ trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java 2007-03-15
20:25:50 UTC (rev 12140)
@@ -203,7 +203,7 @@
tmp.checkForHoles(tmp.countCHKBlocksFromFile(),
false);
- tmp.maybeShrink(true, true);
+ tmp.maybeOfflineShrink(true);
} else {
@@ -566,7 +566,7 @@
System.out.println("Keys in store: db
"+chkBlocksInDatabase+" file "+chkBlocksFromFile+" / max "+maxChkBlocks);
if(!noCheck) {
- maybeShrink(dontCheckForHolesShrinking, true);
+ maybeOfflineShrink(dontCheckForHolesShrinking);
chkBlocksFromFile = countCHKBlocksFromFile();
chkBlocksInStore = Math.max(chkBlocksInStore,
chkBlocksFromFile);
}
@@ -629,23 +629,51 @@
private Object shrinkLock = new Object();
private boolean shrinking = false;
- private void maybeShrink(boolean dontCheckForHoles, boolean offline)
throws DatabaseException, IOException {
- try {
- synchronized(shrinkLock) { if(shrinking) return;
shrinking = true; };
- if(chkBlocksInStore <= maxChkBlocks) return;
- if(offline)
- maybeSlowShrink(dontCheckForHoles, offline);
- else {
- if(chkBlocksInStore * 0.9 > maxChkBlocks) {
- Logger.error(this, "Doing quick and
indiscriminate online shrink. Offline shrinks will preserve the LRU, this
doesn't.");
- maybeQuickShrink(offline);
- } else {
- Logger.error(this, "Online shrink only
supported for small deltas because online shrink does not preserve LRU order.
Suggest you restart the node.");
+ /**
+ * Do an offline shrink, if necessary. Will not return until completed.
+ * @param dontCheckForHoles If true, don't check for holes.
+ * @throws DatabaseException
+ * @throws IOException
+ */
+ private void maybeOfflineShrink(boolean dontCheckForHoles) throws
DatabaseException, IOException {
+ if(chkBlocksInStore <= maxChkBlocks) return;
+ maybeSlowShrink(dontCheckForHoles, true);
+ }
+
+ /**
+ * Do an online shrink, if necessary. Non-blocking i.e. it will do the
shrink on another thread.
+ * @param forceBigOnlineShrinks If true, force the node to shrink the
store immediately even if
+ * it is a major (more than 10%) shrink. Normally this is not allowed
because online shrinks do not
+ * preserve the most recently used data; the best thing to do is to
restart the node and let it do
+ * an offline shrink.
+ * @throws DatabaseException If a database error occurs.
+ * @throws IOException If an I/O error occurs.
+ * @return True if the database will be shrunk in the background (or
the database is already small
+ * enough), false if it is not possible to shrink it because a large
shrink was requested and we
+ * don't want to do a large online shrink.
+ */
+ private boolean maybeOnlineShrink(boolean forceBigOnlineShrinks) throws
DatabaseException, IOException {
+ synchronized(this) {
+ if(chkBlocksInStore <= maxChkBlocks) return true;
+ }
+ if(chkBlocksInStore * 0.9 > maxChkBlocks ||
forceBigOnlineShrinks) {
+ Runnable r = new Runnable() {
+ public void run() {
+ try {
+ synchronized(shrinkLock) {
if(shrinking) return; shrinking = true; };
+ maybeQuickShrink(false);
+ } catch (Throwable t) {
+ Logger.error(this, "Online
shrink failed: "+t, t);
+ } finally {
+ synchronized(shrinkLock) {
shrinking = false; };
+ }
}
- }
- } finally {
- synchronized(shrinkLock) { shrinking = false; };
- }
+ };
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ t.start();
+ return true;
+ } else return false;
}
private void maybeSlowShrink(boolean dontCheckForHoles, boolean
inStartUp) throws DatabaseException, IOException {
@@ -1107,7 +1135,7 @@
lastRecentlyUsed = getMaxRecentlyUsed();
if(!noCheck) {
- maybeShrink(true, true);
+ maybeOfflineShrink(true);
}
// Add shutdownhook
@@ -2170,25 +2198,11 @@
}
}
- public void setMaxKeys(long maxStoreKeys, boolean shrinkNow) throws
DatabaseException, IOException {
+ public void setMaxKeys(long maxStoreKeys, boolean forceBigShrink)
throws DatabaseException, IOException {
synchronized(this) {
maxChkBlocks = maxStoreKeys;
}
- if(shrinkNow) {
- Thread t = new Thread(new Runnable() {
- public void run() {
- try {
- maybeShrink(true, false);
- } catch (DatabaseException e) {
- Logger.error(this, "Cannot
shrink: "+e, e);
- } catch (IOException e) {
- Logger.error(this, "Cannot
shrink: "+e, e);
- }
- }
- });
- t.setDaemon(true);
- t.start();
- }
+ maybeOnlineShrink(false);
}
public long getMaxKeys() {
