Repository: hbase Updated Branches: refs/heads/0.98 bcf3665d2 -> 616f91f1f
HBASE-12891 concurrent region consistency checks Signed-off-by: Andrew Purtell <apurt...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/616f91f1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/616f91f1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/616f91f1 Branch: refs/heads/0.98 Commit: 616f91f1f7f13fdf14662bd023d635f9b595f3d8 Parents: bcf3665 Author: Dave Latham <davelat...@yahoo-inc.com> Authored: Thu Apr 2 09:26:58 2015 -0700 Committer: Andrew Purtell <apurt...@apache.org> Committed: Thu Apr 2 12:14:05 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 73 +++++++++++++++++++- 1 file changed, 70 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/616f91f1/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 4d21043..beda67a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -124,8 +124,10 @@ import org.apache.zookeeper.KeeperException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; +import com.google.common.collect.Ordering; import com.google.common.collect.TreeMultimap; import com.google.protobuf.ServiceException; @@ -1659,8 +1661,56 @@ public class HBaseFsck extends Configured { */ private void checkAndFixConsistency() throws IOException, KeeperException, InterruptedException { + List<CheckRegionConsistencyWorkItem> workItems = + new ArrayList<CheckRegionConsistencyWorkItem>(regionInfoMap.size()); for (java.util.Map.Entry<String, HbckInfo> e: regionInfoMap.entrySet()) { - checkRegionConsistency(e.getKey(), e.getValue()); + workItems.add(new CheckRegionConsistencyWorkItem(e.getKey(), e.getValue())); + } + checkRegionConsistencyConcurrently(workItems); + } + + /** + * Check consistency of all regions using mulitple threads concurrently. + */ + private void checkRegionConsistencyConcurrently( + final List<CheckRegionConsistencyWorkItem> workItems) + throws IOException, KeeperException, InterruptedException { + if (workItems.isEmpty()) { + return; // nothing to check + } + + List<Future<Void>> workFutures = executor.invokeAll(workItems); + for(Future<Void> f: workFutures) { + try { + f.get(); + } catch(ExecutionException e1) { + LOG.warn("Could not check region consistency " , e1.getCause()); + if (e1.getCause() instanceof IOException) { + throw (IOException)e1.getCause(); + } else if (e1.getCause() instanceof KeeperException) { + throw (KeeperException)e1.getCause(); + } else if (e1.getCause() instanceof InterruptedException) { + throw (InterruptedException)e1.getCause(); + } else { + throw new IOException(e1.getCause()); + } + } + } + } + + class CheckRegionConsistencyWorkItem implements Callable<Void> { + private final String key; + private final HbckInfo hbi; + + CheckRegionConsistencyWorkItem(String key, HbckInfo hbi) { + this.key = key; + this.hbi = hbi; + } + + @Override + public synchronized Void call() throws Exception { + checkRegionConsistency(key, hbi); + return null; } } @@ -1917,7 +1967,7 @@ public class HBaseFsck extends Configured { } Collections.sort(tableInfo.regionsFromMeta); } - for (HRegionInfo region : tableInfo.regionsFromMeta) { + for (HRegionInfo region : tableInfo.getRegionsFromMeta()) { if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) <= 0 && (region.getEndKey().length == 0 || Bytes.compareTo(region.getEndKey(), hri.getEndKey()) >= 0) @@ -2264,7 +2314,7 @@ public class HBaseFsck extends Configured { TreeMultimap.create(RegionSplitCalculator.BYTES_COMPARATOR, cmp); // list of regions derived from meta entries. - final List<HRegionInfo> regionsFromMeta = new ArrayList<HRegionInfo>(); + private ImmutableList<HRegionInfo> regionsFromMeta = null; TableInfo(TableName name) { this.tableName = name; @@ -2319,6 +2369,23 @@ public class HBaseFsck extends Configured { return sc.getStarts().size() + backwards.size(); } + public synchronized ImmutableList<HRegionInfo> getRegionsFromMeta() { + // lazy loaded, synchronized to ensure a single load + if (regionsFromMeta == null) { + List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); + for (HbckInfo h : HBaseFsck.this.regionInfoMap.values()) { + if (tableName.equals(h.getTableName())) { + if (h.metaEntry != null) { + regions.add((HRegionInfo) h.metaEntry); + } + } + } + regionsFromMeta = Ordering.natural().immutableSortedCopy(regions); + } + + return regionsFromMeta; + } + private class IntegrityFixSuggester extends TableIntegrityErrorHandlerImpl { ErrorReporter errors;
