Repository: hbase Updated Branches: refs/heads/branch-1 8be6f95f9 -> 8691520e3
HBASE-16399 Provide an API to get list of failed regions and servername in Canary (Vishal Khandelwal) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8691520e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8691520e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8691520e Branch: refs/heads/branch-1 Commit: 8691520e335aa9a2001c21aac5248bf7daed2139 Parents: 8be6f95 Author: Andrew Purtell <apurt...@apache.org> Authored: Fri Sep 2 10:13:35 2016 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Fri Sep 2 10:43:20 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/tool/Canary.java | 64 ++++++++++++++++++-- .../hadoop/hbase/tool/TestCanaryTool.java | 12 ++-- 2 files changed, 66 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/8691520e/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java index 1e27dcb..4e87c8e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/Canary.java @@ -39,6 +39,7 @@ import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -113,11 +114,15 @@ public final class Canary implements Tool { public long incReadFailureCount(); public void publishReadFailure(HRegionInfo region, Exception e); public void publishReadFailure(HRegionInfo region, HColumnDescriptor column, Exception e); + public void updateReadFailedHostList(HRegionInfo region, String serverName); + public Map<String,String> getReadFailures(); public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime); public long getWriteFailureCount(); public void publishWriteFailure(HRegionInfo region, Exception e); public void publishWriteFailure(HRegionInfo region, HColumnDescriptor column, Exception e); public void publishWriteTiming(HRegionInfo region, HColumnDescriptor column, long msTime); + public void updateWriteFailedHostList(HRegionInfo region, String serverName); + public Map<String,String> getWriteFailures(); } // new extended sink for output regionserver mode info // do not change the Sink interface directly due to maintaining the API @@ -132,6 +137,9 @@ public final class Canary implements Tool { private AtomicLong readFailureCount = new AtomicLong(0), writeFailureCount = new AtomicLong(0); + private Map<String, String> readFailures = new ConcurrentHashMap<String, String>(); + private Map<String, String> writeFailures = new ConcurrentHashMap<String, String>(); + @Override public long getReadFailureCount() { return readFailureCount.get(); @@ -156,9 +164,24 @@ public final class Canary implements Tool { } @Override + public void updateReadFailedHostList(HRegionInfo region, String serverName) { + readFailures.put(region.getRegionNameAsString(), serverName); + } + + @Override public void publishReadTiming(HRegionInfo region, HColumnDescriptor column, long msTime) { LOG.info(String.format("read from region %s column family %s in %dms", - region.getRegionNameAsString(), column.getNameAsString(), msTime)); + region.getRegionNameAsString(), column.getNameAsString(), msTime)); + } + + @Override + public Map<String, String> getReadFailures() { + return readFailures; + } + + @Override + public Map<String, String> getWriteFailures() { + return writeFailures; } @Override @@ -184,6 +207,12 @@ public final class Canary implements Tool { LOG.info(String.format("write to region %s column family %s in %dms", region.getRegionNameAsString(), column.getNameAsString(), msTime)); } + + @Override + public void updateWriteFailedHostList(HRegionInfo region, String serverName) { + writeFailures.put(region.getRegionNameAsString(), serverName); + } + } // a ExtendedSink implementation public static class RegionServerStdOutSink extends StdOutSink implements ExtendedSink { @@ -263,11 +292,13 @@ public final class Canary implements Tool { private Sink sink; private TaskType taskType; private boolean rawScanEnabled; + private ServerName serverName; - RegionTask(Connection connection, HRegionInfo region, Sink sink, TaskType taskType, - boolean rawScanEnabled) { + RegionTask(Connection connection, HRegionInfo region, ServerName serverName, Sink sink, + TaskType taskType, boolean rawScanEnabled) { this.connection = connection; this.region = region; + this.serverName = serverName; this.sink = sink; this.taskType = taskType; this.rawScanEnabled = rawScanEnabled; @@ -354,6 +385,7 @@ public final class Canary implements Tool { sink.publishReadTiming(region, column, stopWatch.getTime()); } catch (Exception e) { sink.publishReadFailure(region, column, e); + sink.updateReadFailedHostList(region, serverName.getHostname()); } finally { if (rs != null) { rs.close(); @@ -410,6 +442,7 @@ public final class Canary implements Tool { table.close(); } catch (IOException e) { sink.publishWriteFailure(region, e); + sink.updateWriteFailedHostList(region, serverName.getHostname()); } return null; } @@ -731,6 +764,14 @@ public final class Canary implements Tool { return monitor.errorCode; } + public Map<String, String> getReadFailures() { + return sink.getReadFailures(); + } + + public Map<String, String> getWriteFailures() { + return sink.getWriteFailures(); + } + private void printUsageAndExit() { System.err.printf( "Usage: bin/hbase %s [opts] [table1 [table2]...] | [regionserver1 [regionserver2]..]%n", @@ -1145,14 +1186,25 @@ public final class Canary implements Tool { table = admin.getConnection().getTable(tableDesc.getTableName()); } catch (TableNotFoundException e) { return new ArrayList<Future<Void>>(); + } finally { + if (table != null) { + table.close(); + } } + List<RegionTask> tasks = new ArrayList<RegionTask>(); + RegionLocator regionLocator = null; try { - for (HRegionInfo region : admin.getTableRegions(tableDesc.getTableName())) { - tasks.add(new RegionTask(admin.getConnection(), region, sink, taskType, rawScanEnabled)); + regionLocator = admin.getConnection().getRegionLocator(tableDesc.getTableName()); + for (HRegionLocation location : regionLocator.getAllRegionLocations()) { + ServerName rs = location.getServerName(); + HRegionInfo region = location.getRegionInfo(); + tasks.add(new RegionTask(admin.getConnection(), region, rs, sink, taskType, rawScanEnabled)); } } finally { - table.close(); + if (regionLocator != null) { + regionLocator.close(); + } } return executor.invokeAll(tasks); } http://git-wip-us.apache.org/repos/asf/hbase/blob/8691520e/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java index b5b0d37..f910668 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/tool/TestCanaryTool.java @@ -54,6 +54,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.never; +import static org.junit.Assert.assertEquals; @RunWith(MockitoJUnitRunner.class) @Category({MediumTests.class}) @@ -111,10 +112,11 @@ public class TestCanaryTool { ExecutorService executor = new ScheduledThreadPoolExecutor(1); Canary.RegionServerStdOutSink sink = spy(new Canary.RegionServerStdOutSink()); Canary canary = new Canary(executor, sink); - String[] args = { "-t", "10000", "testTable" }; + String[] args = { "-writeSniffing", "-t", "10000", "testTable" }; ToolRunner.run(testingUtility.getConfiguration(), canary, args); - verify(sink, atLeastOnce()) - .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); + assertEquals("verify no read error count", 0, canary.getReadFailures().size()); + assertEquals("verify no write error count", 0, canary.getWriteFailures().size()); + verify(sink, atLeastOnce()).publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); } //no table created, so there should be no regions @@ -163,13 +165,15 @@ public class TestCanaryTool { ToolRunner.run(conf, canary, args); verify(sink, atLeastOnce()) .publishReadTiming(isA(HRegionInfo.class), isA(HColumnDescriptor.class), anyLong()); + assertEquals("verify no read error count", 0, canary.getReadFailures().size()); } - + private void runRegionserverCanary() throws Exception { ExecutorService executor = new ScheduledThreadPoolExecutor(1); Canary canary = new Canary(executor, new Canary.RegionServerStdOutSink()); String[] args = { "-t", "10000", "-regionserver"}; ToolRunner.run(testingUtility.getConfiguration(), canary, args); + assertEquals("verify no read error count", 0, canary.getReadFailures().size()); } }
