hbase git commit: HBASE-16570 Compute region locality in parallel at startup (binlijin)

[chenheng]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 493c31c29 -> a5f0223bd


HBASE-16570 Compute region locality in parallel at startup (binlijin)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a5f0223b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a5f0223b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a5f0223b

Branch: refs/heads/branch-1.3
Commit: a5f0223bd1db25c18c11ba1250f53066e50f28e8
Parents: 493c31c
Author: chenheng 
Authored: Fri Sep 9 10:54:48 2016 +0800
Committer: chenheng 
Committed: Fri Sep 9 14:19:48 2016 +0800

--
 .../hbase/master/balancer/BaseLoadBalancer.java | 49 +++
 .../master/balancer/RegionLocationFinder.java   | 18 +--
 .../master/balancer/TestBaseLoadBalancer.java   | 51 +---
 .../balancer/TestRegionLocationFinder.java  |  3 +-
 4 files changed, 100 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a5f0223b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index c2529a8..0c86557 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -33,6 +33,7 @@ import java.util.NavigableMap;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.logging.Log;
@@ -57,6 +58,7 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * The base class for load balancers. It provides the the functions used to by
@@ -115,6 +117,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 HRegionInfo[] regions;
 Deque[] regionLoads;
 private RegionLocationFinder regionFinder;
+ArrayList regionLocationFutures;
 
 int[][] regionLocations; //regionIndex -> list of serverIndex sorted by 
locality
 
@@ -236,6 +239,13 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   regionIndexToTableIndex = new int[numRegions];
   regionIndexToPrimaryIndex = new int[numRegions];
   regionLoads = new Deque[numRegions];
+  regionLocationFutures = new 
ArrayList(
+  numRegions);
+  if (regionFinder != null) {
+for (int i = 0; i < numRegions; i++) {
+  regionLocationFutures.add(null);
+}
+  }
   regionLocations = new int[numRegions][];
   serverIndicesSortedByRegionCount = new Integer[numServers];
   serverIndicesSortedByLocality = new Integer[numServers];
@@ -305,6 +315,33 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 regionIndex++;
   }
 
+  if (regionFinder != null) {
+for (int index = 0; index < regionLocationFutures.size(); index++) {
+  ListenableFuture future = 
regionLocationFutures
+  .get(index);
+  HDFSBlocksDistribution blockDistbn = null;
+  try {
+blockDistbn = future.get();
+  } catch (InterruptedException ite) {
+  } catch (ExecutionException ee) {
+LOG.debug(
+"IOException during HDFSBlocksDistribution computation. for 
region = "
++ regions[index].getEncodedName(), ee);
+  } finally {
+if (blockDistbn == null) {
+  blockDistbn = new HDFSBlocksDistribution();
+}
+  }
+  List loc = 
regionFinder.getTopBlockLocations(blockDistbn);
+  regionLocations[index] = new int[loc.size()];
+  for (int i = 0; i < loc.size(); i++) {
+regionLocations[index][i] = loc.get(i) == null ? -1
+: (serversToIndex.get(loc.get(i).getHostAndPort()) == null ? -1
+: serversToIndex.get(loc.get(i).getHostAndPort()));
+  }
+}
+  }
+
   for (int i = 0; i < serversPerHostList.size(); i++) {
 serversPerHost[i] = new int[serversPerHostList.get(i).size()];
 for (int j = 0; j < serversPerHost[i].length; j++) {
@@ -452,15 +489,9 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   }
 
   if (regionFinder != null) {
-

hbase git commit: HBASE-16570 Compute region locality in parallel at startup (binlijin)

[chenheng]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 52963b342 -> b4086795f


HBASE-16570 Compute region locality in parallel at startup (binlijin)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b4086795
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b4086795
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b4086795

Branch: refs/heads/branch-1
Commit: b4086795f2dcb1497a367592850fa80f6514cde2
Parents: 52963b3
Author: chenheng 
Authored: Fri Sep 9 10:54:48 2016 +0800
Committer: chenheng 
Committed: Fri Sep 9 14:16:37 2016 +0800

--
 .../hbase/master/balancer/BaseLoadBalancer.java | 49 +++
 .../master/balancer/RegionLocationFinder.java   | 18 +--
 .../master/balancer/TestBaseLoadBalancer.java   | 51 +---
 .../balancer/TestRegionLocationFinder.java  |  3 +-
 4 files changed, 100 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b4086795/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 0a61839..93b29b6 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -33,6 +33,7 @@ import java.util.NavigableMap;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.logging.Log;
@@ -57,6 +58,7 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * The base class for load balancers. It provides the the functions used to by
@@ -115,6 +117,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 HRegionInfo[] regions;
 Deque[] regionLoads;
 private RegionLocationFinder regionFinder;
+ArrayList regionLocationFutures;
 
 int[][] regionLocations; //regionIndex -> list of serverIndex sorted by 
locality
 
@@ -236,6 +239,13 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   regionIndexToTableIndex = new int[numRegions];
   regionIndexToPrimaryIndex = new int[numRegions];
   regionLoads = new Deque[numRegions];
+  regionLocationFutures = new 
ArrayList(
+  numRegions);
+  if (regionFinder != null) {
+for (int i = 0; i < numRegions; i++) {
+  regionLocationFutures.add(null);
+}
+  }
   regionLocations = new int[numRegions][];
   serverIndicesSortedByRegionCount = new Integer[numServers];
   serverIndicesSortedByLocality = new Integer[numServers];
@@ -305,6 +315,33 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 regionIndex++;
   }
 
+  if (regionFinder != null) {
+for (int index = 0; index < regionLocationFutures.size(); index++) {
+  ListenableFuture future = 
regionLocationFutures
+  .get(index);
+  HDFSBlocksDistribution blockDistbn = null;
+  try {
+blockDistbn = future.get();
+  } catch (InterruptedException ite) {
+  } catch (ExecutionException ee) {
+LOG.debug(
+"IOException during HDFSBlocksDistribution computation. for 
region = "
++ regions[index].getEncodedName(), ee);
+  } finally {
+if (blockDistbn == null) {
+  blockDistbn = new HDFSBlocksDistribution();
+}
+  }
+  List loc = 
regionFinder.getTopBlockLocations(blockDistbn);
+  regionLocations[index] = new int[loc.size()];
+  for (int i = 0; i < loc.size(); i++) {
+regionLocations[index][i] = loc.get(i) == null ? -1
+: (serversToIndex.get(loc.get(i).getHostAndPort()) == null ? -1
+: serversToIndex.get(loc.get(i).getHostAndPort()));
+  }
+}
+  }
+
   for (int i = 0; i < serversPerHostList.size(); i++) {
 serversPerHost[i] = new int[serversPerHostList.get(i).size()];
 for (int j = 0; j < serversPerHost[i].length; j++) {
@@ -452,15 +489,9 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   }
 
   if (regionFinder != null) {
-

hbase git commit: HBASE-16570 Compute region locality in parallel at startup (binlijin)

[chenheng]

Repository: hbase
Updated Branches:
  refs/heads/master 46c756a4a -> e11aafae9


HBASE-16570 Compute region locality in parallel at startup (binlijin)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e11aafae
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e11aafae
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e11aafae

Branch: refs/heads/master
Commit: e11aafae957bc8d71cb622833011f29325049987
Parents: 46c756a
Author: chenheng 
Authored: Fri Sep 9 10:54:48 2016 +0800
Committer: chenheng 
Committed: Fri Sep 9 10:54:48 2016 +0800

--
 .../hbase/master/balancer/BaseLoadBalancer.java | 49 +++
 .../master/balancer/RegionLocationFinder.java   | 18 +--
 .../master/balancer/TestBaseLoadBalancer.java   | 51 +---
 .../balancer/TestRegionLocationFinder.java  |  3 +-
 4 files changed, 100 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e11aafae/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index dc5bace..2b13b21 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -33,6 +33,7 @@ import java.util.NavigableMap;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.commons.logging.Log;
@@ -59,6 +60,7 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * The base class for load balancers. It provides the the functions used to by
@@ -117,6 +119,7 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 HRegionInfo[] regions;
 Deque[] regionLoads;
 private RegionLocationFinder regionFinder;
+ArrayList regionLocationFutures;
 
 int[][] regionLocations; //regionIndex -> list of serverIndex sorted by 
locality
 
@@ -238,6 +241,13 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   regionIndexToTableIndex = new int[numRegions];
   regionIndexToPrimaryIndex = new int[numRegions];
   regionLoads = new Deque[numRegions];
+  regionLocationFutures = new 
ArrayList(
+  numRegions);
+  if (regionFinder != null) {
+for (int i = 0; i < numRegions; i++) {
+  regionLocationFutures.add(null);
+}
+  }
   regionLocations = new int[numRegions][];
   serverIndicesSortedByRegionCount = new Integer[numServers];
   serverIndicesSortedByLocality = new Integer[numServers];
@@ -307,6 +317,33 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 regionIndex++;
   }
 
+  if (regionFinder != null) {
+for (int index = 0; index < regionLocationFutures.size(); index++) {
+  ListenableFuture future = 
regionLocationFutures
+  .get(index);
+  HDFSBlocksDistribution blockDistbn = null;
+  try {
+blockDistbn = future.get();
+  } catch (InterruptedException ite) {
+  } catch (ExecutionException ee) {
+LOG.debug(
+"IOException during HDFSBlocksDistribution computation. for 
region = "
++ regions[index].getEncodedName(), ee);
+  } finally {
+if (blockDistbn == null) {
+  blockDistbn = new HDFSBlocksDistribution();
+}
+  }
+  List loc = 
regionFinder.getTopBlockLocations(blockDistbn);
+  regionLocations[index] = new int[loc.size()];
+  for (int i = 0; i < loc.size(); i++) {
+regionLocations[index][i] = loc.get(i) == null ? -1
+: (serversToIndex.get(loc.get(i).getHostAndPort()) == null ? -1
+: serversToIndex.get(loc.get(i).getHostAndPort()));
+  }
+}
+  }
+
   for (int i = 0; i < serversPerHostList.size(); i++) {
 serversPerHost[i] = new int[serversPerHostList.get(i).size()];
 for (int j = 0; j < serversPerHost[i].length; j++) {
@@ -454,15 +491,9 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
   }
 
   if (regionFinder != null) {
-