Author: robert
Date: 2007-12-12 21:00:27 +0000 (Wed, 12 Dec 2007)
New Revision: 16508
Modified:
trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/node/NodeStats.java
Log:
Rough implementation of cache & store location statistics (uses a lot of memory)
Modified: trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
2007-12-12 18:16:09 UTC (rev 16507)
+++ trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
2007-12-12 21:00:27 UTC (rev 16508)
@@ -29,6 +29,7 @@
import freenet.support.SizeUtil;
import freenet.support.TimeUtil;
import freenet.support.api.HTTPRequest;
+import freenet.support.math.SimpleRunningAverage;
public class StatisticsToadlet extends Toadlet {
@@ -422,8 +423,52 @@
"\u00a0(" + ((storeHits*100) /
(storeAccesses)) + "%)");
storeSizeList.addChild("li",
- "Avg. access rate:\u00a0" +
thousendPoint.format(overallAccesses/nodeUptimeSeconds) + "/sec");
+ "Avg. access rate:\u00a0" +
thousendPoint.format(cacheAccesses/nodeUptimeSeconds) + "/sec,
"+thousendPoint.format(storeAccesses/nodeUptimeSeconds)+"/sec");
+ // location-based stats
+ boolean hasLoc=true;
+ double nodeLoc=0.0;
+ try {
+ nodeLoc=node.getLocationManager().getLocation();
+ } catch (Error e) {
+ //FIXME: PLEASE, how do we get the node location on the
stats page?
+ //Logger.error(this, "why?", e);
+ e.printStackTrace();
+ hasLoc=false;
+ }
+ double
avgCacheLocation=node.nodeStats.avgCacheLocation.currentValue();
+ double
avgStoreLocation=node.nodeStats.avgStoreLocation.currentValue();
+ long cacheWrites=node.nodeStats.avgCacheLocation.countReports();
+ long storeWrites=node.nodeStats.avgStoreLocation.countReports();
+ double
avgCacheSuccess=node.nodeStats.avgCacheSuccess.currentValue();
+ double
avgStoreSuccess=node.nodeStats.avgStoreSuccess.currentValue();
+ double furthestCacheSuccess=node.nodeStats.furthestCacheSuccess;
+ double furthestStoreSuccess=node.nodeStats.furthestStoreSuccess;
+ double storeDist=Location.distance(nodeLoc, avgStoreLocation);
+ double cacheDist=Location.distance(nodeLoc, avgCacheLocation);
+
+ storeSizeList.addChild("li", "avgCacheLocation:\u00a0" +
thousendPoint.format(avgCacheLocation));
+ storeSizeList.addChild("li", "avgStoreLocation:\u00a0" +
thousendPoint.format(avgStoreLocation));
+
+ storeSizeList.addChild("li", "avgCacheSuccess:\u00a0" +
thousendPoint.format(avgCacheSuccess));
+ storeSizeList.addChild("li", "avgStoreSuccess:\u00a0" +
thousendPoint.format(avgStoreSuccess));
+
+ storeSizeList.addChild("li", "furthestCacheSuccess:\u00a0" +
thousendPoint.format(furthestCacheSuccess));
+ storeSizeList.addChild("li", "furthestStoreSuccess:\u00a0" +
thousendPoint.format(furthestStoreSuccess));
+
+ storeSizeList.addChild("li", "cacheWrites:\u00a0" +
cacheWrites);
+ storeSizeList.addChild("li", "storeWrites:\u00a0" +
storeWrites);
+
+ if (hasLoc) {
+ storeSizeList.addChild("li", "cacheDist:\u00a0" +
thousendPoint.format(cacheDist));
+ storeSizeList.addChild("li", "storeDist:\u00a0" +
thousendPoint.format(storeDist));
+ long
cacheLocationReports=((SimpleRunningAverage)node.nodeStats.avgCacheLocation).countReports();
+ long
storeLocationReports=((SimpleRunningAverage)node.nodeStats.avgStoreLocation).countReports();
+ double
cachePrimePercent=((1.0*cacheLocationReports)/cachedKeys);
+ double
storePrimePercent=((1.0*storeLocationReports)/storeKeys);
+ storeSizeList.addChild("li",
"locStatsReliability:\u00a0"+fix3p1pct.format(cachePrimePercent)+" /
"+fix3p1pct.format(storePrimePercent));
+ }
+
}
private void drawUnclaimedFIFOMessageCountsBox(HTMLNode
unclaimedFIFOMessageCountsInfobox) {
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2007-12-12 18:16:09 UTC (rev
16507)
+++ trunk/freenet/src/freenet/node/Node.java 2007-12-12 21:00:27 UTC (rev
16508)
@@ -247,8 +247,8 @@
/** The maximum number of keys stored in each of the datastores, cache
and store combined. */
private long maxTotalKeys;
- private long maxCacheKeys;
- private long maxStoreKeys;
+ long maxCacheKeys;
+ long maxStoreKeys;
/** The maximum size of the datastore. Kept to avoid rounding turning
5G into 5368698672 */
private long maxTotalDatastoreSize;
/** If true, store shrinks occur immediately even if they are over 10%
of the store size. If false,
@@ -1767,11 +1767,23 @@
public SSKBlock fetch(NodeSSK key, boolean dontPromote) {
if(logMINOR) dumpStoreHits();
try {
+ double loc=key.toNormalizedDouble();
+ double dist=Location.distance(lm.getLocation(), loc);
+ nodeStats.avgRequestLocation.report(loc);
SSKBlock block = sskDatastore.fetch(key, dontPromote);
if(block != null) {
+ nodeStats.avgStoreSuccess.report(loc);
+ if (dist > nodeStats.furthestStoreSuccess)
+ nodeStats.furthestStoreSuccess=dist;
return block;
}
- return sskDatacache.fetch(key, dontPromote);
+ block=sskDatacache.fetch(key, dontPromote);
+ if (block != null) {
+ nodeStats.avgCacheSuccess.report(loc);
+ if (dist > nodeStats.furthestCacheSuccess)
+ nodeStats.furthestCacheSuccess=dist;
+ }
+ return block;
} catch (IOException e) {
Logger.error(this, "Cannot fetch data: "+e, e);
return null;
@@ -1781,9 +1793,23 @@
public CHKBlock fetch(NodeCHK key, boolean dontPromote) {
if(logMINOR) dumpStoreHits();
try {
+ double loc=key.toNormalizedDouble();
+ double dist=Location.distance(lm.getLocation(), loc);
+ nodeStats.avgRequestLocation.report(loc);
CHKBlock block = chkDatastore.fetch(key, dontPromote);
- if(block != null) return block;
- return chkDatacache.fetch(key, dontPromote);
+ if (block != null) {
+ nodeStats.avgStoreSuccess.report(loc);
+ if (dist > nodeStats.furthestStoreSuccess)
+ nodeStats.furthestStoreSuccess=dist;
+ return block;
+ }
+ block=chkDatacache.fetch(key, dontPromote);
+ if (block != null) {
+ nodeStats.avgCacheSuccess.report(loc);
+ if (dist > nodeStats.furthestCacheSuccess)
+ nodeStats.furthestCacheSuccess=dist;
+ }
+ return block;
} catch (IOException e) {
Logger.error(this, "Cannot fetch data: "+e, e);
return null;
@@ -1835,10 +1861,13 @@
private void store(CHKBlock block, boolean deep) {
try {
+ double loc=block.getKey().toNormalizedDouble();
if(deep) {
chkDatastore.put(block);
+ nodeStats.avgStoreLocation.report(loc);
}
chkDatacache.put(block);
+ nodeStats.avgCacheLocation.report(loc);
if(clientCore != null && clientCore.requestStarters !=
null)
clientCore.requestStarters.chkFetchScheduler.tripPendingKey(block);
} catch (IOException e) {
@@ -2498,6 +2527,10 @@
return usm;
}
+ public LocationManager getLocationManager() {
+ return lm;
+ }
+
public int getSwaps() {
return LocationManager.swaps;
}
Modified: trunk/freenet/src/freenet/node/NodeStats.java
===================================================================
--- trunk/freenet/src/freenet/node/NodeStats.java 2007-12-12 18:16:09 UTC
(rev 16507)
+++ trunk/freenet/src/freenet/node/NodeStats.java 2007-12-12 21:00:27 UTC
(rev 16508)
@@ -22,6 +22,7 @@
import freenet.support.api.IntCallback;
import freenet.support.api.LongCallback;
import freenet.support.math.RunningAverage;
+import freenet.support.math.SimpleRunningAverage;
import freenet.support.math.TimeDecayingRunningAverage;
import freenet.support.math.TrivialRunningAverage;
@@ -144,8 +145,17 @@
// various metrics
public final RunningAverage routingMissDistance;
public final RunningAverage backedOffPercent;
+ public final RunningAverage avgCacheLocation;
+ public final RunningAverage avgStoreLocation;
+ public final RunningAverage avgCacheSuccess;
+ public final RunningAverage avgStoreSuccess;
+ // FIXME: does furthest{Store,Cache}Success need to be synchronized?
+ public double furthestCacheSuccess=0.0;
+ public double furthestStoreSuccess=0.0;
protected final Persister persister;
+ protected final RunningAverage avgRequestLocation;
+
// ThreadCounting stuffs
public final ThreadGroup rootThreadGroup;
private int threadLimit;
@@ -176,6 +186,14 @@
this.hardRandom = node.random;
this.routingMissDistance = new TimeDecayingRunningAverage(0.0,
180000, 0.0, 1.0, node);
this.backedOffPercent = new TimeDecayingRunningAverage(0.0,
180000, 0.0, 1.0, node);
+ // FIXME PLEASE remove (int) casts
+ double nodeLoc=node.lm.getLocation();
+ this.avgCacheLocation=new
SimpleRunningAverage((int)node.maxCacheKeys, nodeLoc);
+ this.avgStoreLocation=new
SimpleRunningAverage((int)node.maxStoreKeys, nodeLoc);
+ // FIXME average for success-location may not need to be so
large as the store.
+ this.avgCacheSuccess=new SimpleRunningAverage(10000, nodeLoc);
+ this.avgStoreSuccess=new SimpleRunningAverage(10000, nodeLoc);
+ this.avgRequestLocation=new SimpleRunningAverage(10000,
nodeLoc);
preemptiveRejectReasons = new StringCounter();
localPreemptiveRejectReasons = new StringCounter();
pInstantRejectIncoming = new TimeDecayingRunningAverage(0,
60000, 0.0, 1.0, node);
@@ -186,6 +204,18 @@
new TimeDecayingRunningAverage(1, 10*60*1000 /* should
be significantly longer than a typical transfer */, 0, Long.MAX_VALUE, node);
nodePinger = new NodePinger(node);
+ // FIXME: data-store/cache averages need to be persisted to be
valuable (or scanned at every launch).
+ /*
+ if (node.isAdvancedModeEnabled()) {
+ //Uggghh....
+ System.err.println("Scanning datastore/cache for
location values");
+ chkDatastore.kludgeScan(avgStoreLocation);
+ sskDatastore.kludgeScan(avgStoreLocation);
+ chkDatacache.kludgeScan(avgCacheLocation);
+ sskDatacache.kludgeScan(avgCacheLocation);
+ }
+ */
+
previous_input_stat = 0;
previous_output_stat = 0;
previous_io_stat_time = 1;
