[1/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE during split transaction

2018-12-05 Thread apurtell
Repository: hbase
Updated Branches:
  refs/heads/branch-1.4 dab7b39f2 -> d625b212e
Updated Tags:  refs/tags/1.4.9RC1 [created] c5b401c4f


HBASE-21464 Splitting blocked with meta NSRE during split transaction

Signed-off-by: Lars Hofhansl 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7c79c72
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7c79c72
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7c79c72

Branch: refs/heads/branch-1.4
Commit: e7c79c722376aa152dae6f976c01faebefed224f
Parents: dab7b39
Author: Andrew Purtell 
Authored: Fri Nov 30 15:23:34 2018 -0800
Committer: Andrew Purtell 
Committed: Wed Dec 5 10:54:59 2018 -0800

--
 .../hadoop/hbase/client/ConnectionManager.java  | 42 
 1 file changed, 25 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e7c79c72/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 7cf09c2..35ffa3e 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -1231,38 +1231,46 @@ class ConnectionManager {
   }
 }
 
+private volatile RegionLocations metaLocations = null;
+private volatile long lastMetaLookupTime = 
EnvironmentEdgeManager.currentTime();
+// cache meta location at most 10 seconds
+private final static long META_LOOKUP_CACHE_INTERVAL = 1;
+
 private RegionLocations locateMeta(final TableName tableName,
 boolean useCache, int replicaId) throws IOException {
-  // HBASE-10785: We cache the location of the META itself, so that we are 
not overloading
-  // zookeeper with one request for every region lookup. We cache the META 
with empty row
-  // key in MetaCache.
-  byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the 
row for meta
-  RegionLocations locations = null;
+  // We cache the location of the META itself, so that we are not 
overloading
+  // zookeeper with one request for every region lookup. If relocating, 
bypass
+  // the cache immediately.
   if (useCache) {
-locations = getCachedLocation(tableName, metaCacheKey);
-if (locations != null && locations.getRegionLocation(replicaId) != 
null) {
-  return locations;
+long now = EnvironmentEdgeManager.currentTime();
+if (now - lastMetaLookupTime < META_LOOKUP_CACHE_INTERVAL) {
+  if (metaLocations != null &&
+  metaLocations.getRegionLocation(replicaId) != null) {
+return metaLocations;
+  }
+} else {
+  useCache = false;
 }
   }
-
   // only one thread should do the lookup.
   synchronized (metaRegionLock) {
 // Check the cache again for a hit in case some other thread made the
 // same query while we were waiting on the lock.
 if (useCache) {
-  locations = getCachedLocation(tableName, metaCacheKey);
-  if (locations != null && locations.getRegionLocation(replicaId) != 
null) {
-return locations;
+  if (metaLocations != null &&
+  metaLocations.getRegionLocation(replicaId) != null) {
+return metaLocations;
   }
 }
-
 // Look up from zookeeper
-locations = this.registry.getMetaRegionLocation();
-if (locations != null) {
-  cacheLocation(tableName, locations);
+metaLocations = this.registry.getMetaRegionLocation();
+lastMetaLookupTime = EnvironmentEdgeManager.currentTime();
+if (metaLocations != null &&
+metaLocations.getRegionLocation(replicaId) != null) {
+  return metaLocations;
 }
+return null;
   }
-  return locations;
 }
 
 /*



[1/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE during split transaction

2018-11-30 Thread apurtell
Repository: hbase
Updated Branches:
  refs/heads/branch-1 f1e2f077f -> bd87f4ebc
  refs/heads/branch-1.4 302edf447 -> 70d9934e4


HBASE-21464 Splitting blocked with meta NSRE during split transaction

When looking up the locations of hbase:meta with useCache false, clear all 
previous
cache entries for it first

Fix Table reference leaks in MetaTableAccessor with try-with-resources

Signed-off-by: Allan Yang 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd87f4eb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd87f4eb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd87f4eb

Branch: refs/heads/branch-1
Commit: bd87f4ebcd49e9a3d100bba81fd6ab8868027c06
Parents: f1e2f07
Author: Andrew Purtell 
Authored: Thu Nov 29 15:46:21 2018 -0800
Committer: Andrew Purtell 
Committed: Fri Nov 30 09:57:54 2018 -0800

--
 .../apache/hadoop/hbase/MetaTableAccessor.java  | 95 +---
 .../hadoop/hbase/client/ConnectionManager.java  | 10 ++-
 2 files changed, 49 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bd87f4eb/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 440f8c6..fd7a97b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -175,8 +175,7 @@ public class MetaTableAccessor {
* @return An {@link Table} for hbase:meta
* @throws IOException
*/
-  static Table getMetaHTable(final Connection connection)
-  throws IOException {
+  static Table getMetaHTable(final Connection connection) throws IOException {
 // We used to pass whole CatalogTracker in here, now we just pass in 
Connection
 if (connection == null) {
   throw new NullPointerException("No connection");
@@ -248,11 +247,13 @@ public class MetaTableAccessor {
 }
 Get get = new Get(row);
 get.addFamily(HConstants.CATALOG_FAMILY);
-Result r = get(getMetaHTable(connection), get);
-RegionLocations locations = getRegionLocations(r);
-return locations == null
-  ? null
-  : locations.getRegionLocation(parsedInfo == null ? 0 : 
parsedInfo.getReplicaId());
+try (Table metaTable = getMetaHTable(connection)) {
+  Result r = get(metaTable, get);
+  RegionLocations locations = getRegionLocations(r);
+  return locations == null
+  ? null
+  : locations.getRegionLocation(parsedInfo == null ? 0 : 
parsedInfo.getReplicaId());
+}
   }
 
   /**
@@ -267,8 +268,10 @@ public class MetaTableAccessor {
 byte[] row = getMetaKeyForRegion(regionInfo);
 Get get = new Get(row);
 get.addFamily(HConstants.CATALOG_FAMILY);
-Result r = get(getMetaHTable(connection), get);
-return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+try (Table metaTable = getMetaHTable(connection)) {
+  Result r = get(metaTable, get);
+  return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
+}
   }
 
   /** Returns the row key to use for this regionInfo */
@@ -300,7 +303,9 @@ public class MetaTableAccessor {
   byte[] regionName) throws IOException {
 Get get = new Get(regionName);
 get.addFamily(HConstants.CATALOG_FAMILY);
-return get(getMetaHTable(connection), get);
+try (Table metaTable = getMetaHTable(connection)) {
+  return get(metaTable, get);
+}
   }
 
   /**
@@ -631,19 +636,19 @@ public class MetaTableAccessor {
   scan.setCaching(caching);
 }
 scan.addFamily(HConstants.CATALOG_FAMILY);
-Table metaTable = getMetaHTable(connection);
-ResultScanner scanner = null;
-try {
-  scanner = metaTable.getScanner(scan);
-  Result data;
-  while((data = scanner.next()) != null) {
-if (data.isEmpty()) continue;
-// Break if visit returns false.
-if (!visitor.visit(data)) break;
+try (Table metaTable = getMetaHTable(connection)) {
+  try (ResultScanner scanner = metaTable.getScanner(scan)) {
+Result data;
+while ((data = scanner.next()) != null) {
+  if (data.isEmpty()) {
+continue;
+  }
+  // Break if visit returns false.
+  if (!visitor.visit(data)) {
+break;
+  }
+}
   }
-} finally {
-  if (scanner != null) scanner.close();
-  metaTable.close();
 }
   }
 
@@ -1020,7 +1025,9 @@ public class MetaTableAccessor {
*/
   static void putToMetaTable(final Connection connection, final Put p)
 throws IOException {
-