Author: stack
Date: Mon Mar 14 23:38:57 2011
New Revision: 1081610
URL: http://svn.apache.org/viewvc?rev=1081610&view=rev
Log:
HBASE-3507 requests count per HRegion and rebalance command; part 1
Modified:
hbase/trunk/pom.xml
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Modified: hbase/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/hbase/trunk/pom.xml?rev=1081610&r1=1081609&r2=1081610&view=diff
==============================================================================
--- hbase/trunk/pom.xml (original)
+++ hbase/trunk/pom.xml Mon Mar 14 23:38:57 2011
@@ -556,6 +556,11 @@
<version>${commons-cli.version}</version>
</dependency>
<dependency>
+ <groupId>com.github.stephenc.high-scale-lib</groupId>
+ <artifactId>high-scale-lib</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons-codec.version}</version>
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1081610&r1=1081609&r2=1081610&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Mon Mar
14 23:38:57 2011
@@ -65,6 +65,8 @@ public class HServerLoad implements Writ
private int memstoreSizeMB;
/** the current total size of storefile indexes for the region, in MB */
private int storefileIndexSizeMB;
+ /** the current total request made to region */
+ private long requestsCount;
/**
* Constructor, for Writable
@@ -80,16 +82,19 @@ public class HServerLoad implements Writ
* @param storefileSizeMB
* @param memstoreSizeMB
* @param storefileIndexSizeMB
+ * @param requestsCount
*/
public RegionLoad(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
- final int memstoreSizeMB, final int storefileIndexSizeMB) {
+ final int memstoreSizeMB, final int storefileIndexSizeMB,
+ final long requestsCount) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
+ this.requestsCount = requestsCount;
}
// Getters
@@ -143,6 +148,13 @@ public class HServerLoad implements Writ
return storefileIndexSizeMB;
}
+ /**
+ * @return the number of requests made to region
+ */
+ public long getRequestsCount() {
+ return requestsCount;
+ }
+
// Setters
/**
@@ -181,6 +193,13 @@ public class HServerLoad implements Writ
this.storefileIndexSizeMB = storefileIndexSizeMB;
}
+ /**
+ * @param requestsCount the number of requests to region
+ */
+ public void setRequestsCount(long requestsCount) {
+ this.requestsCount = requestsCount;
+ }
+
// Writable
public void readFields(DataInput in) throws IOException {
int namelen = in.readInt();
@@ -191,6 +210,7 @@ public class HServerLoad implements Writ
this.storefileSizeMB = in.readInt();
this.memstoreSizeMB = in.readInt();
this.storefileIndexSizeMB = in.readInt();
+ this.requestsCount = in.readLong();
}
public void write(DataOutput out) throws IOException {
@@ -201,6 +221,7 @@ public class HServerLoad implements Writ
out.writeInt(storefileSizeMB);
out.writeInt(memstoreSizeMB);
out.writeInt(storefileIndexSizeMB);
+ out.writeLong(requestsCount);
}
/**
@@ -218,6 +239,8 @@ public class HServerLoad implements Writ
Integer.valueOf(this.memstoreSizeMB));
sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
Integer.valueOf(this.storefileIndexSizeMB));
+ sb = Strings.appendKeyValue(sb, "requestsCount",
+ Long.valueOf(this.requestsCount));
return sb.toString();
}
}
@@ -452,14 +475,16 @@ public class HServerLoad implements Writ
* @param storefiles
* @param memstoreSizeMB
* @param storefileIndexSizeMB
+ * @param requestsCount
* @deprecated Use {@link #addRegionInfo(RegionLoad)}
*/
@Deprecated
public void addRegionInfo(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
- final int memstoreSizeMB, final int storefileIndexSizeMB) {
+ final int memstoreSizeMB, final int storefileIndexSizeMB,
+ final long requestsCount) {
this.regionLoad.add(new HServerLoad.RegionLoad(name, stores, storefiles,
- storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB));
+ storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount));
}
// Writable
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1081610&r1=1081609&r2=1081610&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Mon Mar 14 23:38:57 2011
@@ -93,9 +93,10 @@ import org.apache.hadoop.hbase.util.FSUt
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
+import org.cliffc.high_scale_lib.Counter;
+
import com.google.common.collect.ClassToInstanceMap;
import com.google.common.collect.Lists;
import com.google.common.collect.MutableClassToInstanceMap;
@@ -174,6 +175,8 @@ public class HRegion implements HeapSize
final AtomicLong memstoreSize = new AtomicLong(0);
+ final Counter requestsCount = new Counter();
+
/**
* The directory for the table this region is part of.
* This directory contains the directory for this region.
@@ -455,6 +458,11 @@ public class HRegion implements HeapSize
return this.regionInfo;
}
+ /** @return requestsCount for this region */
+ public long getRequestsCount() {
+ return this.requestsCount.get();
+ }
+
/** @return true if region is closed */
public boolean isClosed() {
return this.closed.get();
@@ -2962,6 +2970,7 @@ public class HRegion implements HeapSize
listPaths(fs, newRegionDir);
}
HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf,
newRegionInfo, null);
+ dstRegion.requestsCount.set(a.requestsCount.get() + b.requestsCount.get());
dstRegion.initialize();
dstRegion.compactStores();
if (LOG.isDebugEnabled()) {
@@ -3371,7 +3380,7 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
(4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
- (24 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
+ (25 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
(ClassSize.OBJECT * 2) + (2 * ClassSize.ATOMIC_BOOLEAN) +
@@ -3634,6 +3643,7 @@ public class HRegion implements HeapSize
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closed");
}
+ this.requestsCount.increment();
}
/**
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1081610&r1=1081609&r2=1081610&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Mon Mar 14 23:38:57 2011
@@ -901,6 +901,7 @@ public class HRegionServer implements HR
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
int storefileIndexSizeMB = 0;
+ long requestsCount = r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Store store : r.stores.values()) {
@@ -909,8 +910,8 @@ public class HRegionServer implements HR
storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 /
1024);
}
}
- return new HServerLoad.RegionLoad(name, stores, storefiles,
- storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB);
+ return new HServerLoad.RegionLoad(name,stores, storefiles,
+ storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount);
}
/**
@@ -1149,11 +1150,13 @@ public class HRegionServer implements HR
int stores = 0;
int storefiles = 0;
long memstoreSize = 0;
+ long requestsCount = 0;
long storefileIndexSize = 0;
synchronized (this.onlineRegions) {
for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
HRegion r = e.getValue();
memstoreSize += r.memstoreSize.get();
+ requestsCount += r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Map.Entry<byte[], Store> ee : r.stores.entrySet()) {
@@ -1167,6 +1170,7 @@ public class HRegionServer implements HR
this.metrics.stores.set(stores);
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
+ this.metrics.requestsCount.set(requestsCount);
this.metrics.storefileIndexSizeMB
.set((int) (storefileIndexSize / (1024 * 1024)));
this.metrics.compactionQueueSize.set(compactSplitThread
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java?rev=1081610&r1=1081609&r2=1081610&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
Mon Mar 14 23:38:57 2011
@@ -543,6 +543,7 @@ public class SplitTransaction {
HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
this.parent.getLog(), fs, this.parent.getConf(),
hri, rsServices);
+ r.requestsCount.set(this.parent.getRequestsCount() / 2);
HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
return r;
}
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1081610&r1=1081609&r2=1081610&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Mon Mar 14 23:38:57 2011
@@ -128,6 +128,11 @@ public class RegionServerMetrics impleme
public final MetricsIntValue storefiles = new MetricsIntValue("storefiles",
registry);
/**
+ * Count of requests
+ */
+ public final MetricsLongValue requestsCount = new
MetricsLongValue("requestsCount", registry);
+
+ /**
* Sum of all the storefile index sizes in this regionserver in MB
*/
public final MetricsIntValue storefileIndexSizeMB =
@@ -243,6 +248,7 @@ public class RegionServerMetrics impleme
this.storefiles.pushMetric(this.metricsRecord);
this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
this.memstoreSizeMB.pushMetric(this.metricsRecord);
+ this.requestsCount.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
this.requests.pushMetric(this.metricsRecord);
this.compactionQueueSize.pushMetric(this.metricsRecord);
@@ -345,6 +351,8 @@ public class RegionServerMetrics impleme
Integer.valueOf(this.storefileIndexSizeMB.get()));
sb = Strings.appendKeyValue(sb, "memstoreSize",
Integer.valueOf(this.memstoreSizeMB.get()));
+ sb = Strings.appendKeyValue(sb, "requestsCount",
+ Long.valueOf(this.requestsCount.get()));
sb = Strings.appendKeyValue(sb, "compactionQueueSize",
Integer.valueOf(this.compactionQueueSize.get()));
sb = Strings.appendKeyValue(sb, "flushQueueSize",
