Okay, this is good on the whole, but you are right, the memory usage is
unacceptable. You should use a cheaper running average;
BootstrappingDecayingRunningAverage is the obvious thing (a klein filter
whose parameter starts off at 1.0 and decays until it reaches a certain
point), but if you want something more conventional, make something that just
keeps counters; you know what value you are removing when a key is removed
from the store.
On Wednesday 12 December 2007 21:00, you wrote:
> Author: robert
> Date: 2007-12-12 21:00:27 +0000 (Wed, 12 Dec 2007)
> New Revision: 16508
>
> Modified:
> trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
> trunk/freenet/src/freenet/node/Node.java
> trunk/freenet/src/freenet/node/NodeStats.java
> Log:
> Rough implementation of cache & store location statistics (uses a lot of
memory)
>
>
> Modified: trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
> ===================================================================
> --- trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
> 2007-12-12
18:16:09 UTC (rev 16507)
> +++ trunk/freenet/src/freenet/clients/http/StatisticsToadlet.java
> 2007-12-12
21:00:27 UTC (rev 16508)
> @@ -29,6 +29,7 @@
> import freenet.support.SizeUtil;
> import freenet.support.TimeUtil;
> import freenet.support.api.HTTPRequest;
> +import freenet.support.math.SimpleRunningAverage;
>
> public class StatisticsToadlet extends Toadlet {
>
> @@ -422,8 +423,52 @@
> "\u00a0(" + ((storeHits*100) /
> (storeAccesses)) + "%)");
>
> storeSizeList.addChild("li",
> - "Avg. access rate:\u00a0" +
thousendPoint.format(overallAccesses/nodeUptimeSeconds) + "/sec");
> + "Avg. access rate:\u00a0" +
thousendPoint.format(cacheAccesses/nodeUptimeSeconds)
+ "/sec, "+thousendPoint.format(storeAccesses/nodeUptimeSeconds)+"/sec");
>
> + // location-based stats
> + boolean hasLoc=true;
> + double nodeLoc=0.0;
> + try {
> + nodeLoc=node.getLocationManager().getLocation();
> + } catch (Error e) {
> + //FIXME: PLEASE, how do we get the node location on the
> stats page?
> + //Logger.error(this, "why?", e);
> + e.printStackTrace();
> + hasLoc=false;
> + }
> + double
> avgCacheLocation=node.nodeStats.avgCacheLocation.currentValue();
> + double
> avgStoreLocation=node.nodeStats.avgStoreLocation.currentValue();
> + long cacheWrites=node.nodeStats.avgCacheLocation.countReports();
> + long storeWrites=node.nodeStats.avgStoreLocation.countReports();
> + double
> avgCacheSuccess=node.nodeStats.avgCacheSuccess.currentValue();
> + double
> avgStoreSuccess=node.nodeStats.avgStoreSuccess.currentValue();
> + double furthestCacheSuccess=node.nodeStats.furthestCacheSuccess;
> + double furthestStoreSuccess=node.nodeStats.furthestStoreSuccess;
> + double storeDist=Location.distance(nodeLoc, avgStoreLocation);
> + double cacheDist=Location.distance(nodeLoc, avgCacheLocation);
> +
> + storeSizeList.addChild("li", "avgCacheLocation:\u00a0" +
thousendPoint.format(avgCacheLocation));
> + storeSizeList.addChild("li", "avgStoreLocation:\u00a0" +
thousendPoint.format(avgStoreLocation));
> +
> + storeSizeList.addChild("li", "avgCacheSuccess:\u00a0" +
thousendPoint.format(avgCacheSuccess));
> + storeSizeList.addChild("li", "avgStoreSuccess:\u00a0" +
thousendPoint.format(avgStoreSuccess));
> +
> + storeSizeList.addChild("li", "furthestCacheSuccess:\u00a0" +
thousendPoint.format(furthestCacheSuccess));
> + storeSizeList.addChild("li", "furthestStoreSuccess:\u00a0" +
thousendPoint.format(furthestStoreSuccess));
> +
> + storeSizeList.addChild("li", "cacheWrites:\u00a0" +
> cacheWrites);
> + storeSizeList.addChild("li", "storeWrites:\u00a0" +
> storeWrites);
> +
> + if (hasLoc) {
> + storeSizeList.addChild("li", "cacheDist:\u00a0" +
thousendPoint.format(cacheDist));
> + storeSizeList.addChild("li", "storeDist:\u00a0" +
thousendPoint.format(storeDist));
> + long
cacheLocationReports=((SimpleRunningAverage)node.nodeStats.avgCacheLocation).countReports();
> + long
storeLocationReports=((SimpleRunningAverage)node.nodeStats.avgStoreLocation).countReports();
> + double
> cachePrimePercent=((1.0*cacheLocationReports)/cachedKeys);
> + double
> storePrimePercent=((1.0*storeLocationReports)/storeKeys);
> + storeSizeList.addChild("li", "locStatsReliability:
\u00a0"+fix3p1pct.format(cachePrimePercent)+" /
"+fix3p1pct.format(storePrimePercent));
> + }
> +
> }
>
> private void drawUnclaimedFIFOMessageCountsBox(HTMLNode
unclaimedFIFOMessageCountsInfobox) {
>
> Modified: trunk/freenet/src/freenet/node/Node.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/Node.java 2007-12-12 18:16:09 UTC (rev
16507)
> +++ trunk/freenet/src/freenet/node/Node.java 2007-12-12 21:00:27 UTC (rev
16508)
> @@ -247,8 +247,8 @@
>
> /** The maximum number of keys stored in each of the datastores, cache
> and
store combined. */
> private long maxTotalKeys;
> - private long maxCacheKeys;
> - private long maxStoreKeys;
> + long maxCacheKeys;
> + long maxStoreKeys;
> /** The maximum size of the datastore. Kept to avoid rounding turning
> 5G
into 5368698672 */
> private long maxTotalDatastoreSize;
> /** If true, store shrinks occur immediately even if they are over 10%
> of
the store size. If false,
> @@ -1767,11 +1767,23 @@
> public SSKBlock fetch(NodeSSK key, boolean dontPromote) {
> if(logMINOR) dumpStoreHits();
> try {
> + double loc=key.toNormalizedDouble();
> + double dist=Location.distance(lm.getLocation(), loc);
> + nodeStats.avgRequestLocation.report(loc);
> SSKBlock block = sskDatastore.fetch(key, dontPromote);
> if(block != null) {
> + nodeStats.avgStoreSuccess.report(loc);
> + if (dist > nodeStats.furthestStoreSuccess)
> + nodeStats.furthestStoreSuccess=dist;
> return block;
> }
> - return sskDatacache.fetch(key, dontPromote);
> + block=sskDatacache.fetch(key, dontPromote);
> + if (block != null) {
> + nodeStats.avgCacheSuccess.report(loc);
> + if (dist > nodeStats.furthestCacheSuccess)
> + nodeStats.furthestCacheSuccess=dist;
> + }
> + return block;
> } catch (IOException e) {
> Logger.error(this, "Cannot fetch data: "+e, e);
> return null;
> @@ -1781,9 +1793,23 @@
> public CHKBlock fetch(NodeCHK key, boolean dontPromote) {
> if(logMINOR) dumpStoreHits();
> try {
> + double loc=key.toNormalizedDouble();
> + double dist=Location.distance(lm.getLocation(), loc);
> + nodeStats.avgRequestLocation.report(loc);
> CHKBlock block = chkDatastore.fetch(key, dontPromote);
> - if(block != null) return block;
> - return chkDatacache.fetch(key, dontPromote);
> + if (block != null) {
> + nodeStats.avgStoreSuccess.report(loc);
> + if (dist > nodeStats.furthestStoreSuccess)
> + nodeStats.furthestStoreSuccess=dist;
> + return block;
> + }
> + block=chkDatacache.fetch(key, dontPromote);
> + if (block != null) {
> + nodeStats.avgCacheSuccess.report(loc);
> + if (dist > nodeStats.furthestCacheSuccess)
> + nodeStats.furthestCacheSuccess=dist;
> + }
> + return block;
> } catch (IOException e) {
> Logger.error(this, "Cannot fetch data: "+e, e);
> return null;
> @@ -1835,10 +1861,13 @@
>
> private void store(CHKBlock block, boolean deep) {
> try {
> + double loc=block.getKey().toNormalizedDouble();
> if(deep) {
> chkDatastore.put(block);
> + nodeStats.avgStoreLocation.report(loc);
> }
> chkDatacache.put(block);
> + nodeStats.avgCacheLocation.report(loc);
> if(clientCore != null && clientCore.requestStarters !=
> null)
>
> clientCore.requestStarters.chkFetchScheduler.tripPendingKey(block);
> } catch (IOException e) {
> @@ -2498,6 +2527,10 @@
> return usm;
> }
>
> + public LocationManager getLocationManager() {
> + return lm;
> + }
> +
> public int getSwaps() {
> return LocationManager.swaps;
> }
>
> Modified: trunk/freenet/src/freenet/node/NodeStats.java
> ===================================================================
> --- trunk/freenet/src/freenet/node/NodeStats.java 2007-12-12 18:16:09 UTC
(rev 16507)
> +++ trunk/freenet/src/freenet/node/NodeStats.java 2007-12-12 21:00:27 UTC
(rev 16508)
> @@ -22,6 +22,7 @@
> import freenet.support.api.IntCallback;
> import freenet.support.api.LongCallback;
> import freenet.support.math.RunningAverage;
> +import freenet.support.math.SimpleRunningAverage;
> import freenet.support.math.TimeDecayingRunningAverage;
> import freenet.support.math.TrivialRunningAverage;
>
> @@ -144,8 +145,17 @@
> // various metrics
> public final RunningAverage routingMissDistance;
> public final RunningAverage backedOffPercent;
> + public final RunningAverage avgCacheLocation;
> + public final RunningAverage avgStoreLocation;
> + public final RunningAverage avgCacheSuccess;
> + public final RunningAverage avgStoreSuccess;
> + // FIXME: does furthest{Store,Cache}Success need to be synchronized?
> + public double furthestCacheSuccess=0.0;
> + public double furthestStoreSuccess=0.0;
> protected final Persister persister;
>
> + protected final RunningAverage avgRequestLocation;
> +
> // ThreadCounting stuffs
> public final ThreadGroup rootThreadGroup;
> private int threadLimit;
> @@ -176,6 +186,14 @@
> this.hardRandom = node.random;
> this.routingMissDistance = new TimeDecayingRunningAverage(0.0,
> 180000,
0.0, 1.0, node);
> this.backedOffPercent = new TimeDecayingRunningAverage(0.0,
> 180000, 0.0,
1.0, node);
> + // FIXME PLEASE remove (int) casts
> + double nodeLoc=node.lm.getLocation();
> + this.avgCacheLocation=new
> SimpleRunningAverage((int)node.maxCacheKeys,
nodeLoc);
> + this.avgStoreLocation=new
> SimpleRunningAverage((int)node.maxStoreKeys,
nodeLoc);
> + // FIXME average for success-location may not need to be so
> large as the
store.
> + this.avgCacheSuccess=new SimpleRunningAverage(10000, nodeLoc);
> + this.avgStoreSuccess=new SimpleRunningAverage(10000, nodeLoc);
> + this.avgRequestLocation=new SimpleRunningAverage(10000,
> nodeLoc);
> preemptiveRejectReasons = new StringCounter();
> localPreemptiveRejectReasons = new StringCounter();
> pInstantRejectIncoming = new TimeDecayingRunningAverage(0,
> 60000, 0.0,
1.0, node);
> @@ -186,6 +204,18 @@
> new TimeDecayingRunningAverage(1, 10*60*1000 /* should
> be significantly
longer than a typical transfer */, 0, Long.MAX_VALUE, node);
> nodePinger = new NodePinger(node);
>
> + // FIXME: data-store/cache averages need to be persisted to be
> valuable
(or scanned at every launch).
> + /*
> + if (node.isAdvancedModeEnabled()) {
> + //Uggghh....
> + System.err.println("Scanning datastore/cache for
> location values");
> + chkDatastore.kludgeScan(avgStoreLocation);
> + sskDatastore.kludgeScan(avgStoreLocation);
> + chkDatacache.kludgeScan(avgCacheLocation);
> + sskDatacache.kludgeScan(avgCacheLocation);
> + }
> + */
> +
> previous_input_stat = 0;
> previous_output_stat = 0;
> previous_io_stat_time = 1;
>
> _______________________________________________
> cvs mailing list
> cvs at freenetproject.org
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>
>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL:
<https://emu.freenetproject.org/pipermail/devl/attachments/20071213/51c98613/attachment.pgp>
