Repository: hbase Updated Branches: refs/heads/0.98 b9e9f0537 -> 695812aab
HBASE-17205 Add a metric for the duration of region in transition Amending-Author: Andrew Purtell <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/695812aa Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/695812aa Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/695812aa Branch: refs/heads/0.98 Commit: 695812aab604192f2cbcfc16c49bb4005129d746 Parents: b9e9f05 Author: Guanghao Zhang <[email protected]> Authored: Thu Dec 1 09:42:43 2016 +0800 Committer: Andrew Purtell <[email protected]> Committed: Tue Dec 6 15:31:01 2016 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hbase/master/RegionState.java | 21 ++++++++++++++++++ .../master/MetricsAssignmentManagerSource.java | 3 +++ .../MetricsAssignmentManagerSourceImpl.java | 7 ++++++ .../MetricsAssignmentManagerSourceImpl.java | 7 ++++++ .../hadoop/hbase/master/AssignmentManager.java | 7 ++++-- .../hbase/master/MetricsAssignmentManager.java | 8 +++++++ .../hadoop/hbase/master/RegionStates.java | 23 +++++++++++++++----- .../apache/hadoop/hbase/TestDrainingServer.java | 5 +++-- .../hbase/master/TestAssignmentManager.java | 15 ++++++------- 9 files changed, 78 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java index ec96196..7a5aff7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java @@ -180,6 +180,8 @@ public class RegionState { private volatile ServerName serverName; private volatile State state; + // The duration of region in transition + private long ritDuration; public RegionState() { this.stamp = new AtomicLong(System.currentTimeMillis()); @@ -196,10 +198,16 @@ public class RegionState { public RegionState(HRegionInfo region, State state, long stamp, ServerName serverName) { + this(region, state, stamp, serverName, 0); + } + + public RegionState(HRegionInfo region, State state, long stamp, ServerName serverName, + long ritDuration) { this.hri = region; this.state = state; this.stamp = new AtomicLong(stamp); this.serverName = serverName; + this.ritDuration = ritDuration; } public void updateTimestampToNow() { @@ -222,6 +230,19 @@ public class RegionState { return serverName; } + public long getRitDuration() { + return ritDuration; + } + + /** + * Update the duration of region in transition + * @param previousStamp previous RegionState's timestamp + */ + @InterfaceAudience.Private + void updateRitDuration(long previousStamp) { + this.ritDuration += (this.stamp.get() - previousStamp); + } + public boolean isClosing() { return state == State.CLOSING; } http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java index 92fd111..08b4ac5 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java @@ -45,6 +45,7 @@ public interface MetricsAssignmentManagerSource extends BaseSource { String RIT_COUNT_NAME = "ritCount"; String RIT_COUNT_OVER_THRESHOLD_NAME = "ritCountOverThreshold"; String RIT_OLDEST_AGE_NAME = "ritOldestAge"; + String RIT_DURATION_NAME = "ritDuration"; String ASSIGN_TIME_NAME = "assign"; String BULK_ASSIGN_TIME_NAME = "bulkAssign"; @@ -72,4 +73,6 @@ public interface MetricsAssignmentManagerSource extends BaseSource { * @param age age of the oldest RIT. */ void setRITOldestAge(long age); + + void updateRitDuration(long duration); } http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java index bb4e748..1d33a94 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java @@ -27,6 +27,7 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement private MetricMutableGaugeLong ritGauge; private MetricMutableGaugeLong ritCountOverThresholdGauge; private MetricMutableGaugeLong ritOldestAgeGauge; + private MetricMutableHistogram ritDurationHisto; private MetricMutableHistogram assignTimeHisto; private MetricMutableHistogram bulkAssignTimeHisto; @@ -48,6 +49,7 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement ritOldestAgeGauge = metricsRegistry.newGauge(RIT_OLDEST_AGE_NAME, "", 0l); assignTimeHisto = metricsRegistry.newTimeHistogram(ASSIGN_TIME_NAME); bulkAssignTimeHisto = metricsRegistry.newTimeHistogram(BULK_ASSIGN_TIME_NAME); + ritDurationHisto = metricsRegistry.newTimeHistogram(RIT_DURATION_NAME); } @Override @@ -71,4 +73,9 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement public void setRITOldestAge(long ritCount) { ritOldestAgeGauge.set(ritCount); } + + @Override + public void updateRitDuration(long duration) { + ritDurationHisto.add(duration); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java index ccf1c1d..38baacf 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java @@ -29,6 +29,7 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement private MutableGaugeLong ritGauge; private MutableGaugeLong ritCountOverThresholdGauge; private MutableGaugeLong ritOldestAgeGauge; + private MutableHistogram ritDurationHisto; private MutableHistogram assignTimeHisto; private MutableHistogram bulkAssignTimeHisto; @@ -48,6 +49,7 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement ritOldestAgeGauge = metricsRegistry.newGauge(RIT_OLDEST_AGE_NAME, "", 0l); assignTimeHisto = metricsRegistry.newTimeHistogram(ASSIGN_TIME_NAME); bulkAssignTimeHisto = metricsRegistry.newTimeHistogram(BULK_ASSIGN_TIME_NAME); + ritDurationHisto = metricsRegistry.newTimeHistogram(RIT_DURATION_NAME); } @Override @@ -71,4 +73,9 @@ public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implement public void setRITOldestAge(long ritCount) { ritOldestAgeGauge.set(ritCount); } + + @Override + public void updateRitDuration(long duration) { + ritDurationHisto.add(duration); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 6100eda..923a48e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -27,7 +27,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; @@ -275,7 +274,7 @@ public class AssignmentManager extends ZooKeeperListener { * @throws KeeperException * @throws IOException */ - public AssignmentManager(Server server, ServerManager serverManager, + public AssignmentManager(MasterServices server, ServerManager serverManager, CatalogTracker catalogTracker, final LoadBalancer balancer, final ExecutorService service, MetricsMaster metricsMaster, final TableLockManager tableLockManager) throws KeeperException, IOException { @@ -343,6 +342,10 @@ public class AssignmentManager extends ZooKeeperListener { } } + MetricsAssignmentManager getAssignmentManagerMetrics() { + return this.metricsAssignmentManager; + } + /** * Add the listener to the notification list. * @param listener The AssignmentListener to register http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java index 7b2423c..8fb4de6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java @@ -60,4 +60,12 @@ public class MetricsAssignmentManager { public void updateRITOldestAge(long timestamp) { assignmentManagerSource.setRITOldestAge(timestamp); } + + /** + * update the duration metrics of region is transition + * @param duration + */ + public void updateRitDuration(long duration) { + assignmentManagerSource.updateRitDuration(duration); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 9c49aa9..47b164c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -35,7 +34,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RegionTransition; -import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -120,13 +118,13 @@ public class RegionStates { private final RegionStateStore regionStateStore; private final ServerManager serverManager; - private final Server server; + private final MasterServices server; // The maximum time to keep a log split info in region states map static final String LOG_SPLIT_TIME = "hbase.master.maximum.logsplit.keeptime"; static final long DEFAULT_LOG_SPLIT_TIME = 7200000L; // 2 hours - RegionStates(final Server master, + RegionStates(final MasterServices master, final ServerManager serverManager, final RegionStateStore regionStateStore) { regionStates = new HashMap<String, RegionState>(); regionsInTransition = new HashMap<String, RegionState>(); @@ -385,7 +383,15 @@ public class RegionStates { updateRegionState(hri, State.OPEN, serverName, openSeqNum); synchronized (this) { - regionsInTransition.remove(hri.getEncodedName()); + RegionState regionState = regionsInTransition.remove(hri.getEncodedName()); + // When region is online and remove from regionsInTransition, + // update the RIT duration to assignment manager metrics + if (regionState != null && this.server.getAssignmentManager() != null) { + long ritDuration = System.currentTimeMillis() - regionState.getStamp() + + regionState.getRitDuration(); + this.server.getAssignmentManager().getAssignmentManagerMetrics() + .updateRitDuration(ritDuration); + } ServerName oldServerName = regionAssignments.put(hri, serverName); if (!serverName.equals(oldServerName)) { LOG.info("Onlined " + hri.getShortNameToLog() + " on " + serverName); @@ -891,7 +897,12 @@ public class RegionStates { } synchronized (this) { - regionsInTransition.put(encodedName, regionState); + RegionState oldRegionState = regionsInTransition.put(encodedName, regionState); + // When region transform old region state to new region state, + // accumulate the RIT duration to new region state. + if (oldRegionState != null) { + regionState.updateRitDuration(oldRegionState.getStamp()); + } putRegionState(regionState); // For these states, region should be properly closed. http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java index c2c90f5..fb0b536 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestDrainingServer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.executor.ExecutorType; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.ServerManager; @@ -89,7 +90,7 @@ public class TestDrainingServer { AssignmentManager am; Configuration conf = TEST_UTIL.getConfiguration(); final HMaster master = Mockito.mock(HMaster.class); - final Server server = Mockito.mock(Server.class); + final MasterServices server = Mockito.mock(MasterServices.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000); @@ -163,7 +164,7 @@ public class TestDrainingServer { CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class); AssignmentManager am; final HMaster master = Mockito.mock(HMaster.class); - final Server server = Mockito.mock(Server.class); + final MasterServices server = Mockito.mock(MasterServices.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000); http://git-wip-us.apache.org/repos/asf/hbase/blob/695812aa/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java index 781dab0..5d289ad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.RegionException; import org.apache.hadoop.hbase.RegionTransition; -import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; @@ -111,7 +110,7 @@ public class TestAssignmentManager { private static boolean enabling = false; // Mocked objects or; get redone for each test. - private Server server; + private MasterServices server; private ServerManager serverManager; private ZooKeeperWatcher watcher; private LoadBalancer balancer; @@ -136,7 +135,7 @@ public class TestAssignmentManager { // Mock a Server. Have it return a legit Configuration and ZooKeeperWatcher. // If abort is called, be sure to fail the test (don't just swallow it // silently as is mockito default). - this.server = Mockito.mock(Server.class); + this.server = Mockito.mock(MasterServices.class); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1")); Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration()); Mockito.when(server.getCatalogTracker()).thenReturn(null); @@ -957,7 +956,7 @@ public class TestAssignmentManager { Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); // To avoid cast exception in DisableTableHandler process. HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); - Server server = new HMaster(HTU.getConfiguration()); + MasterServices server = new HMaster(HTU.getConfiguration()); AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server, this.serverManager); AtomicBoolean gate = new AtomicBoolean(false); @@ -999,7 +998,7 @@ public class TestAssignmentManager { Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); - Server server = new HMaster(HTU.getConfiguration()); + MasterServices server = new HMaster(HTU.getConfiguration()); Whitebox.setInternalState(server, "serverManager", this.serverManager); AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server, this.serverManager); @@ -1036,7 +1035,7 @@ public class TestAssignmentManager { Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); - Server server = new HMaster(HTU.getConfiguration()); + MasterServices server = new HMaster(HTU.getConfiguration()); Whitebox.setInternalState(server, "serverManager", this.serverManager); AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server, this.serverManager); @@ -1166,7 +1165,7 @@ public class TestAssignmentManager { * @throws IOException * @throws KeeperException */ - private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server, + private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final MasterServices server, final ServerManager manager) throws IOException, KeeperException, ServiceException { // We need a mocked catalog tracker. Its used by our AM instance. CatalogTracker ct = Mockito.mock(CatalogTracker.class); @@ -1235,7 +1234,7 @@ public class TestAssignmentManager { AtomicBoolean gate = new AtomicBoolean(true); public AssignmentManagerWithExtrasForTesting( - final Server master, final ServerManager serverManager, + final MasterServices master, final ServerManager serverManager, final CatalogTracker catalogTracker, final LoadBalancer balancer, final ExecutorService service, final TableLockManager tableLockManager) throws KeeperException, IOException {
