Author: kturner
Date: Fri Apr 27 14:20:54 2012
New Revision: 1331438
URL: http://svn.apache.org/viewvc?rev=1331438&view=rev
Log:
ACCUMULO-517 Display iterator seeks and amount of data read by iterator stack
on monitor page
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/system/StatsIterator.java
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TableInfo.java
accumulo/trunk/core/src/main/thrift/master.thrift
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/server/src/main/resources/web/screen.css
Added:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/system/StatsIterator.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/system/StatsIterator.java?rev=1331438&view=auto
==============================================================================
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/system/StatsIterator.java
(added)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/system/StatsIterator.java
Fri Apr 27 14:20:54 2012
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.system;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+
+/**
+ *
+ */
+public class StatsIterator extends WrappingIterator {
+
+ private int numRead = 0;
+ private AtomicLong seekCounter;
+ private AtomicLong readCounter;
+
+ public StatsIterator(SortedKeyValueIterator<Key,Value> source, AtomicLong
seekCounter, AtomicLong readCounter) {
+ super.setSource(source);
+ this.seekCounter = seekCounter;
+ this.readCounter = readCounter;
+ }
+
+ @Override
+ public void next() throws IOException {
+ super.next();
+ numRead++;
+
+ if (numRead % 23 == 0) {
+ readCounter.addAndGet(numRead);
+ numRead = 0;
+ }
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ return new StatsIterator(getSource().deepCopy(env), seekCounter,
readCounter);
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies,
boolean inclusive) throws IOException {
+ super.seek(range, columnFamilies, inclusive);
+ seekCounter.incrementAndGet();
+ if (super.hasTop())
+ numRead = 1;
+ else
+ numRead = 0;
+ }
+
+ public void report() {
+ readCounter.addAndGet(numRead);
+ numRead = 0;
+ }
+}
Modified:
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TableInfo.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TableInfo.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TableInfo.java
(original)
+++
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TableInfo.java
Fri Apr 27 14:20:54 2012
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField MINOR_FIELD_DESC =
new org.apache.thrift.protocol.TField("minor",
org.apache.thrift.protocol.TType.STRUCT, (short)9);
private static final org.apache.thrift.protocol.TField MAJOR_FIELD_DESC =
new org.apache.thrift.protocol.TField("major",
org.apache.thrift.protocol.TType.STRUCT, (short)10);
private static final org.apache.thrift.protocol.TField SCANS_FIELD_DESC =
new org.apache.thrift.protocol.TField("scans",
org.apache.thrift.protocol.TType.STRUCT, (short)11);
+ private static final org.apache.thrift.protocol.TField SCAN_RATE_FIELD_DESC
= new org.apache.thrift.protocol.TField("scanRate",
org.apache.thrift.protocol.TType.DOUBLE, (short)12);
public long recs;
public long recsInMemory;
@@ -46,6 +47,7 @@ import org.slf4j.LoggerFactory;
public Compacting minor;
public Compacting major;
public Compacting scans;
+ public double scanRate;
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -59,7 +61,8 @@ import org.slf4j.LoggerFactory;
QUERY_BYTE_RATE((short)8, "queryByteRate"),
MINOR((short)9, "minor"),
MAJOR((short)10, "major"),
- SCANS((short)11, "scans");
+ SCANS((short)11, "scans"),
+ SCAN_RATE((short)12, "scanRate");
private static final Map<String, _Fields> byName = new HashMap<String,
_Fields>();
@@ -96,6 +99,8 @@ import org.slf4j.LoggerFactory;
return MAJOR;
case 11: // SCANS
return SCANS;
+ case 12: // SCAN_RATE
+ return SCAN_RATE;
default:
return null;
}
@@ -144,7 +149,8 @@ import org.slf4j.LoggerFactory;
private static final int __INGESTBYTERATE_ISSET_ID = 5;
private static final int __QUERYRATE_ISSET_ID = 6;
private static final int __QUERYBYTERATE_ISSET_ID = 7;
- private BitSet __isset_bit_vector = new BitSet(8);
+ private static final int __SCANRATE_ISSET_ID = 8;
+ private BitSet __isset_bit_vector = new BitSet(9);
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData>
metaDataMap;
static {
@@ -171,6 +177,8 @@ import org.slf4j.LoggerFactory;
new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
Compacting.class)));
tmpMap.put(_Fields.SCANS, new
org.apache.thrift.meta_data.FieldMetaData("scans",
org.apache.thrift.TFieldRequirementType.DEFAULT,
new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
Compacting.class)));
+ tmpMap.put(_Fields.SCAN_RATE, new
org.apache.thrift.meta_data.FieldMetaData("scanRate",
org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TableInfo.class,
metaDataMap);
}
@@ -189,7 +197,8 @@ import org.slf4j.LoggerFactory;
double queryByteRate,
Compacting minor,
Compacting major,
- Compacting scans)
+ Compacting scans,
+ double scanRate)
{
this();
this.recs = recs;
@@ -211,6 +220,8 @@ import org.slf4j.LoggerFactory;
this.minor = minor;
this.major = major;
this.scans = scans;
+ this.scanRate = scanRate;
+ setScanRateIsSet(true);
}
/**
@@ -236,6 +247,7 @@ import org.slf4j.LoggerFactory;
if (other.isSetScans()) {
this.scans = new Compacting(other.scans);
}
+ this.scanRate = other.scanRate;
}
public TableInfo deepCopy() {
@@ -263,6 +275,8 @@ import org.slf4j.LoggerFactory;
this.minor = null;
this.major = null;
this.scans = null;
+ setScanRateIsSet(false);
+ this.scanRate = 0.0;
}
public long getRecs() {
@@ -521,6 +535,29 @@ import org.slf4j.LoggerFactory;
}
}
+ public double getScanRate() {
+ return this.scanRate;
+ }
+
+ public TableInfo setScanRate(double scanRate) {
+ this.scanRate = scanRate;
+ setScanRateIsSet(true);
+ return this;
+ }
+
+ public void unsetScanRate() {
+ __isset_bit_vector.clear(__SCANRATE_ISSET_ID);
+ }
+
+ /** Returns true if field scanRate is set (has been assigned a value) and
false otherwise */
+ public boolean isSetScanRate() {
+ return __isset_bit_vector.get(__SCANRATE_ISSET_ID);
+ }
+
+ public void setScanRateIsSet(boolean value) {
+ __isset_bit_vector.set(__SCANRATE_ISSET_ID, value);
+ }
+
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case RECS:
@@ -611,6 +648,14 @@ import org.slf4j.LoggerFactory;
}
break;
+ case SCAN_RATE:
+ if (value == null) {
+ unsetScanRate();
+ } else {
+ setScanRate((Double)value);
+ }
+ break;
+
}
}
@@ -649,6 +694,9 @@ import org.slf4j.LoggerFactory;
case SCANS:
return getScans();
+ case SCAN_RATE:
+ return new Double(getScanRate());
+
}
throw new IllegalStateException();
}
@@ -682,6 +730,8 @@ import org.slf4j.LoggerFactory;
return isSetMajor();
case SCANS:
return isSetScans();
+ case SCAN_RATE:
+ return isSetScanRate();
}
throw new IllegalStateException();
}
@@ -798,6 +848,15 @@ import org.slf4j.LoggerFactory;
return false;
}
+ boolean this_present_scanRate = true;
+ boolean that_present_scanRate = true;
+ if (this_present_scanRate || that_present_scanRate) {
+ if (!(this_present_scanRate && that_present_scanRate))
+ return false;
+ if (this.scanRate != that.scanRate)
+ return false;
+ }
+
return true;
}
@@ -924,6 +983,16 @@ import org.slf4j.LoggerFactory;
return lastComparison;
}
}
+ lastComparison =
Boolean.valueOf(isSetScanRate()).compareTo(typedOther.isSetScanRate());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetScanRate()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.scanRate,
typedOther.scanRate);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -1029,6 +1098,14 @@ import org.slf4j.LoggerFactory;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
break;
+ case 12: // SCAN_RATE
+ if (field.type == org.apache.thrift.protocol.TType.DOUBLE) {
+ this.scanRate = iprot.readDouble();
+ setScanRateIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
@@ -1083,6 +1160,9 @@ import org.slf4j.LoggerFactory;
this.scans.write(oprot);
oprot.writeFieldEnd();
}
+ oprot.writeFieldBegin(SCAN_RATE_FIELD_DESC);
+ oprot.writeDouble(this.scanRate);
+ oprot.writeFieldEnd();
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -1147,6 +1227,10 @@ import org.slf4j.LoggerFactory;
sb.append(this.scans);
}
first = false;
+ if (!first) sb.append(", ");
+ sb.append("scanRate:");
+ sb.append(this.scanRate);
+ first = false;
sb.append(")");
return sb.toString();
}
Modified: accumulo/trunk/core/src/main/thrift/master.thrift
URL:
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/thrift/master.thrift?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/thrift/master.thrift (original)
+++ accumulo/trunk/core/src/main/thrift/master.thrift Fri Apr 27 14:20:54 2012
@@ -38,6 +38,7 @@ struct TableInfo {
9:Compacting minor;
10:Compacting major;
11:Compacting scans;
+ 12:double scanRate;
}
struct RecoveryStatus {
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
Fri Apr 27 14:20:54 2012
@@ -87,6 +87,7 @@ public class Monitor {
private static double totalIngestRate = 0.0;
private static double totalIngestByteRate = 0.0;
private static double totalQueryRate = 0.0;
+ private static double totalScanRate = 0.0;
private static double totalQueryByteRate = 0.0;
private static long totalEntries = 0L;
private static int totalTabletCount = 0;
@@ -125,6 +126,7 @@ public class Monitor {
private static List<Pair<Long,Integer>> majorCompactionsOverTime =
Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
private static List<Pair<Long,Double>> lookupsOverTime =
Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
private static List<Pair<Long,Integer>> queryRateOverTime =
Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static List<Pair<Long,Integer>> scanRateOverTime =
Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
private static List<Pair<Long,Double>> queryByteRateOverTime =
Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
private static List<Pair<Long,Double>> indexCacheHitRateOverTime =
Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
private static List<Pair<Long,Double>> dataCacheHitRateOverTime =
Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
@@ -186,6 +188,7 @@ public class Monitor {
total.ingestByteRate += more.ingestByteRate;
total.queryRate += more.queryRate;
total.queryByteRate += more.queryByteRate;
+ total.scanRate += more.scanRate;
}
public static TableInfo summarizeTableStats(TabletServerStatus status) {
@@ -260,6 +263,7 @@ public class Monitor {
double totalIngestByteRate = 0.;
double totalQueryRate = 0.;
double totalQueryByteRate = 0.;
+ double totalScanRate = 0.;
long totalEntries = 0;
int totalTabletCount = 0;
int onlineTabletCount = 0;
@@ -310,6 +314,7 @@ public class Monitor {
totalIngestRate += summary.ingestRate;
totalIngestByteRate += summary.ingestByteRate;
totalQueryRate += summary.queryRate;
+ totalScanRate += summary.scanRate;
totalQueryByteRate += summary.queryByteRate;
totalEntries += summary.recs;
totalHoldTime += server.holdTime;
@@ -340,6 +345,7 @@ public class Monitor {
totalIngestByteRate = totalIngestByteRate / 1000000.0;
Monitor.totalIngestByteRate = totalIngestByteRate;
Monitor.totalQueryRate = totalQueryRate;
+ Monitor.totalScanRate = totalScanRate;
totalQueryByteRate = totalQueryByteRate / 1000000.0;
Monitor.totalQueryByteRate = totalQueryByteRate;
Monitor.totalEntries = totalEntries;
@@ -367,6 +373,8 @@ public class Monitor {
queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int)
totalQueryRate));
queryByteRateOverTime.add(new Pair<Long,Double>(currentTime,
totalQueryByteRate));
+ scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int)
totalScanRate));
+
calcCacheHitRate(indexCacheHitRateOverTime, currentTime,
indexCacheHitTracker, indexCacheRequestTracker);
calcCacheHitRate(dataCacheHitRateOverTime, currentTime,
dataCacheHitTracker, dataCacheRequestTracker);
@@ -523,6 +531,10 @@ public class Monitor {
return totalQueryRate;
}
+ public static double getTotalScanRate() {
+ return totalScanRate;
+ }
+
public static double getTotalQueryByteRate() {
return totalQueryByteRate;
}
@@ -603,6 +615,12 @@ public class Monitor {
}
}
+ public static List<Pair<Long,Integer>> getScanRateOverTime() {
+ synchronized (scanRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
+ }
+ }
+
public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
synchronized (queryByteRateOverTime) {
return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
Fri Apr 27 14:20:54 2012
@@ -28,6 +28,8 @@ import java.security.PermissionCollectio
import java.security.Permissions;
import java.security.PrivilegedAction;
import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import javax.servlet.ServletException;
@@ -151,6 +153,10 @@ public class DefaultServlet extends Basi
@SuppressWarnings("unchecked")
private static void plotData(StringBuilder sb, String title,
@SuppressWarnings("rawtypes") List data, boolean points) {
+ plotData(sb, title, points, new ArrayList<String>(), data);
+ }
+
+ private static void plotData(StringBuilder sb, String title, boolean points,
List<String> labels, @SuppressWarnings("rawtypes") List... series) {
sb.append("<div class=\"plotHeading\">");
sb.append(title);
sb.append("</div>");
@@ -160,31 +166,52 @@ public class DefaultServlet extends Basi
sb.append("<script type=\"text/javascript\">\n");
sb.append("$(function () {\n");
- sb.append(" var d1 = [");
- String sep = "";
- for (Pair<Long,? extends Number> point : (List<Pair<Long,? extends
Number>>) data) {
- if (point.getSecond() == null)
- continue;
+ for (int i = 0; i < series.length; i++) {
- String y;
- if (point.getSecond() instanceof Double)
- y = String.format("%1.2f", point.getSecond());
- else
- y = point.getSecond().toString();
+ List<Pair<Long,? extends Number>> data = series[i];
+ sb.append(" var d" + i + " = [");
- sb.append(sep);
- sep = ",";
- sb.append("[" + point.getFirst() + "," + y + "]");
+ String sep = "";
+ for (Pair<Long,? extends Number> point : data) {
+ if (point.getSecond() == null)
+ continue;
+
+ String y;
+ if (point.getSecond() instanceof Double)
+ y = String.format("%1.2f", point.getSecond());
+ else
+ y = point.getSecond().toString();
+
+ sb.append(sep);
+ sep = ",";
+ sb.append("[" + point.getFirst() + "," + y + "]");
+ }
+ sb.append(" ];\n");
}
String opts = "lines: { show: true }";
if (points)
opts = "points: { show: true, radius: 1 }";
- sb.append(" ];\n");
- sb.append(" $.plot($(\"#" + id + "\"), [{ data: d1, " + opts
- + ", color:\"red\" }], {yaxis:{}, xaxis:{mode:\"time\",minTickSize:
[1, \"minute\"],timeformat: \"%H:%M\", ticks:3}});");
+
+ sb.append(" $.plot($(\"#" + id + "\"),");
+ String sep = "";
+
+ String colors[] = new String[] {"red", "blue", "green", "black"};
+
+ sb.append("[");
+ for (int i = 0; i < series.length; i++) {
+ sb.append(sep);
+ sep = ",";
+ sb.append("{ ");
+ if (labels.size() > 0) {
+ sb.append("label: \"" + labels.get(i) + "\", ");
+ }
+ sb.append("data: d" + i + ", " + opts + ", color:\"" + colors[i] + "\"
}");
+ }
+ sb.append("], ");
+ sb.append("{yaxis:{}, xaxis:{mode:\"time\",minTickSize: [1,
\"minute\"],timeformat: \"%H:%M\", ticks:3}});");
sb.append(" });\n");
sb.append("</script>\n");
}
@@ -223,7 +250,7 @@ public class DefaultServlet extends Basi
sb.append("<tr><td>\n");
plotData(sb, "Ingest (Entries/s)", Monitor.getIngestRateOverTime(), false);
sb.append("</td><td>\n");
- plotData(sb, "Scan (Entries/s)", Monitor.getQueryRateOverTime(), false);
+ plotData(sb, "Scan (Entries/s)", false, Arrays.asList("Read", "Returned"),
Monitor.getScanRateOverTime(), Monitor.getQueryRateOverTime());
sb.append("</td></tr>\n");
sb.append("<tr><td>\n");
@@ -235,7 +262,7 @@ public class DefaultServlet extends Basi
sb.append("<tr><td>\n");
plotData(sb, "Load Average", Monitor.getLoadOverTime(), false);
sb.append("</td><td>\n");
- plotData(sb, "Scan Sessions", Monitor.getLookupsOverTime(), false);
+ plotData(sb, "Seeks", Monitor.getLookupsOverTime(), false);
sb.append("</td></tr>\n");
sb.append("<tr><td>\n");
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/MasterServlet.java
Fri Apr 27 14:20:54 2012
@@ -140,7 +140,9 @@ public class MasterServlet extends Basic
masterStatus.addSortableColumn("Entries", new NumberType<Long>(), "The
total number of key/value pairs in Accumulo");
masterStatus.addSortableColumn("Ingest", new NumberType<Long>(), "The
number of Key/Value pairs inserted, per second. "
+ " Note that deleted records are \"inserted\" and will make the
ingest " + "rate increase in the near-term.");
- masterStatus.addSortableColumn("Query", new NumberType<Long>(), "The
total number of Key/Value pairs returned as a result of scans.");
+ masterStatus.addSortableColumn("Entries<br />Read", new
NumberType<Long>(),
+ "The total number of Key/Value pairs read on the server side. Not
all may be returned because of filtering.");
+ masterStatus.addSortableColumn("Entries<br />Returned", new
NumberType<Long>(), "The total number of Key/Value pairs returned as a result
of scans.");
masterStatus.addSortableColumn("Hold Time", new DurationType(0l,
0l), "The maximum amount of time that ingest has been held "
+ "across all servers due to a lack of memory to store the records");
masterStatus.addSortableColumn("OS Load", new
NumberType<Double>(0., guessHighLoad * 1., 0., guessHighLoad * 3.),
@@ -154,6 +156,7 @@ public class MasterServlet extends Basic
row.add(Monitor.getMmi().unassignedTablets);
row.add(Monitor.getTotalEntries());
row.add(Math.round(Monitor.getTotalIngestRate()));
+ row.add(Math.round(Monitor.getTotalScanRate()));
row.add(Math.round(Monitor.getTotalQueryRate()));
row.add(Monitor.getTotalHoldTime());
row.add(ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage());
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/TablesServlet.java
Fri Apr 27 14:20:54 2012
@@ -89,7 +89,9 @@ public class TablesServlet extends Basic
tableList.addSortableColumn("Entries<br />In Memory", new
NumberType<Long>(),
"The total number of key/value pairs stored in memory and not yet
written to disk");
tableList.addSortableColumn("Ingest", new NumberType<Long>(), "The number
of Key/Value pairs inserted. Note that deletes are 'inserted'.");
- tableList.addSortableColumn("Query", new NumberType<Long>(),
+ tableList.addSortableColumn("Entries<br/>Read", new NumberType<Long>(),
+ "The number of Key/Value pairs read on the server side. Not all key
values read may be returned to client because of filtering.");
+ tableList.addSortableColumn("Entries<br/>Returned", new NumberType<Long>(),
"The number of Key/Value pairs returned to clients during queries.
This is <b>not</b> the number of scans.");
tableList.addSortableColumn("Hold Time", new DurationType(0l, 0l),
"The amount of time that ingest operations are suspended while waiting
for data to be written to disk.");
@@ -127,6 +129,7 @@ public class TablesServlet extends Basic
row.add(tableInfo == null ? null : tableInfo.recs);
row.add(tableInfo == null ? null : tableInfo.recsInMemory);
row.add(tableInfo == null ? null : tableInfo.ingestRate);
+ row.add(tableInfo == null ? null : tableInfo.scanRate);
row.add(tableInfo == null ? null : tableInfo.queryRate);
row.add(holdTime.longValue());
row.add(tableInfo);
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Fri Apr 27 14:20:54 2012
@@ -80,6 +80,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.system.MultiIterator;
import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
import
org.apache.accumulo.core.iterators.system.SourceSwitchingIterator.DataSource;
+import org.apache.accumulo.core.iterators.system.StatsIterator;
import org.apache.accumulo.core.iterators.system.VisibilityFilter;
import org.apache.accumulo.core.master.thrift.TabletLoadState;
import org.apache.accumulo.core.security.Authorizations;
@@ -430,6 +431,12 @@ public class Tablet {
private volatile long numEntries;
private volatile long numEntriesInMemory;
+
+ private AtomicLong seekCount = new AtomicLong(0);
+ // a count of the amount of data read by the iterators
+ private AtomicLong scannedCount = new AtomicLong(0);
+ private Rate scannedRate = new Rate(0.2);
+
private ConfigurationObserver configObserver;
private TabletServer tabletServer;
@@ -1953,6 +1960,7 @@ public class Tablet {
private List<MemoryIterator> memIters = null;
private long fileReservationId;
private AtomicBoolean interruptFlag;
+ private StatsIterator statsIterator;
ScanOptions options;
@@ -2051,7 +2059,9 @@ public class Tablet {
TabletIteratorEnvironment iterEnv = new
TabletIteratorEnvironment(IteratorScope.scan, acuTableConf, fileManager, files);
- DeletingIterator delIter = new DeletingIterator(multiIter, false);
+ statsIterator = new StatsIterator(multiIter, seekCount, scannedCount);
+
+ DeletingIterator delIter = new DeletingIterator(statsIterator, false);
ColumnFamilySkippingIterator cfsi = new
ColumnFamilySkippingIterator(delIter);
@@ -2083,6 +2093,10 @@ public class Tablet {
fileManager = null;
}
+ if (statsIterator != null) {
+ statsIterator.report();
+ }
+
}
public void interrupt() {
@@ -3339,6 +3353,10 @@ public class Tablet {
this.numEntries = numEntries;
}
+ public long getNumSeeks() {
+ return seekCount.get();
+ }
+
public long getNumEntries() {
return numEntries;
}
@@ -3527,6 +3545,10 @@ public class Tablet {
return ingestByteRate.rate();
}
+ public double scanRate() {
+ return scannedRate.rate();
+ }
+
public long totalQueries() {
return this.queryCount;
}
@@ -3541,6 +3563,7 @@ public class Tablet {
queryByteRate.update(now, queryBytes);
ingestRate.update(now, ingestCount);
ingestByteRate.update(now, ingestBytes);
+ scannedRate.update(now, scannedCount.get());
}
public long getSplitCreationTime() {
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Fri Apr 27 14:20:54 2012
@@ -211,7 +211,6 @@ public class TabletServer extends Abstra
private static HashMap<String,Long> prevGcTime = new HashMap<String,Long>();
private static long lastMemorySize = 0;
private static long gcTimeIncreasedCount;
- private static AtomicLong scanCount = new AtomicLong();
private static final Class<? extends LoggerStrategy> DEFAULT_LOGGER_STRATEGY
= RoundRobinLoggerStrategy.class;
private static final long MAX_TIME_TO_WAIT_FOR_SCAN_RESULT_MILLIS = 1000;
@@ -1076,8 +1075,6 @@ public class TabletServer extends Abstra
throw e.asThriftException();
}
- scanCount.addAndGet(1);
-
KeyExtent extent = new KeyExtent(textent);
// wait for any writes that are in flight.. this done to ensure
@@ -1286,7 +1283,6 @@ public class TabletServer extends Abstra
sessionManager.unreserveSession(sid);
}
- scanCount.addAndGet(batch.size());
return new InitialMultiScan(sid, result);
}
@@ -3012,6 +3008,9 @@ public class TabletServer extends Abstra
onlineTabletsCopy = new HashMap<KeyExtent,Tablet>(this.onlineTablets);
}
Map<String,TableInfo> tables = new HashMap<String,TableInfo>();
+
+ long seeks = 0;
+
for (Entry<KeyExtent,Tablet> entry : onlineTabletsCopy.entrySet()) {
String tableId = entry.getKey().getTableId().toString();
TableInfo table = tables.get(tableId);
@@ -3030,6 +3029,7 @@ public class TabletServer extends Abstra
table.queryByteRate += tablet.queryByteRate();
table.ingestRate += tablet.ingestRate();
table.ingestByteRate += tablet.ingestByteRate();
+ table.scanRate += tablet.scanRate();
long recsInMemory = tablet.getNumEntriesInMemory();
table.recsInMemory += recsInMemory;
if (tablet.minorCompactionRunning())
@@ -3040,6 +3040,7 @@ public class TabletServer extends Abstra
table.major.running++;
if (tablet.majorCompactionQueued())
table.major.queued++;
+ seeks += tablet.getNumSeeks();
}
for (Entry<String,MapCounter<ScanRunState>> entry : scanCounts.entrySet())
{
@@ -3079,7 +3080,7 @@ public class TabletServer extends Abstra
result.osLoad =
ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
result.name = getClientAddressString();
result.holdTime = resourceManager.holdTime();
- result.lookups = scanCount.get();
+ result.lookups = seeks;
result.loggers = new HashSet<String>();
result.indexCacheHits =
resourceManager.getIndexCache().getStats().getHitCount();
result.indexCacheRequest =
resourceManager.getIndexCache().getStats().getRequestCount();
Modified: accumulo/trunk/server/src/main/resources/web/screen.css
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/resources/web/screen.css?rev=1331438&r1=1331437&r2=1331438&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/resources/web/screen.css (original)
+++ accumulo/trunk/server/src/main/resources/web/screen.css Fri Apr 27 14:20:54
2012
@@ -166,6 +166,19 @@ table {
border: 1px #333333 solid;
}
+/*
+The following is to avoid a problem with the flot
+javascript library. This file sets min-width to 60%
+for all tables. This make the html generated by
+flot.js not work. Flot creates a class called legend
+used in a div that has a child table. This sets
+min-width to 0 for that child table enabling it
+to render correctly.
+*/
+div.legend > table {
+ min-width: 0%;
+}
+
table.sortable {
border-left: 0;
border-right: 0;
