hbase git commit: HBASE-21464 Splitting blocked with meta NSRE during split transaction
Repository: hbase Updated Branches: refs/heads/branch-1 333cd1972 -> e80cc3286 HBASE-21464 Splitting blocked with meta NSRE during split transaction Signed-off-by: Lars Hofhansl Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e80cc328 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e80cc328 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e80cc328 Branch: refs/heads/branch-1 Commit: e80cc3286c30c8184f826ffae6ae1c78c9e31719 Parents: 333cd19 Author: Andrew Purtell Authored: Fri Nov 30 15:23:34 2018 -0800 Committer: Andrew Purtell Committed: Wed Dec 5 10:54:35 2018 -0800 -- .../hadoop/hbase/client/ConnectionManager.java | 42 1 file changed, 25 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e80cc328/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java index 7cf09c2..35ffa3e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java @@ -1231,38 +1231,46 @@ class ConnectionManager { } } +private volatile RegionLocations metaLocations = null; +private volatile long lastMetaLookupTime = EnvironmentEdgeManager.currentTime(); +// cache meta location at most 10 seconds +private final static long META_LOOKUP_CACHE_INTERVAL = 1; + private RegionLocations locateMeta(final TableName tableName, boolean useCache, int replicaId) throws IOException { - // HBASE-10785: We cache the location of the META itself, so that we are not overloading - // zookeeper with one request for every region lookup. We cache the META with empty row - // key in MetaCache. - byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the row for meta - RegionLocations locations = null; + // We cache the location of the META itself, so that we are not overloading + // zookeeper with one request for every region lookup. If relocating, bypass + // the cache immediately. if (useCache) { -locations = getCachedLocation(tableName, metaCacheKey); -if (locations != null && locations.getRegionLocation(replicaId) != null) { - return locations; +long now = EnvironmentEdgeManager.currentTime(); +if (now - lastMetaLookupTime < META_LOOKUP_CACHE_INTERVAL) { + if (metaLocations != null && + metaLocations.getRegionLocation(replicaId) != null) { +return metaLocations; + } +} else { + useCache = false; } } - // only one thread should do the lookup. synchronized (metaRegionLock) { // Check the cache again for a hit in case some other thread made the // same query while we were waiting on the lock. if (useCache) { - locations = getCachedLocation(tableName, metaCacheKey); - if (locations != null && locations.getRegionLocation(replicaId) != null) { -return locations; + if (metaLocations != null && + metaLocations.getRegionLocation(replicaId) != null) { +return metaLocations; } } - // Look up from zookeeper -locations = this.registry.getMetaRegionLocation(); -if (locations != null) { - cacheLocation(tableName, locations); +metaLocations = this.registry.getMetaRegionLocation(); +lastMetaLookupTime = EnvironmentEdgeManager.currentTime(); +if (metaLocations != null && +metaLocations.getRegionLocation(replicaId) != null) { + return metaLocations; } +return null; } - return locations; } /*
[1/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE during split transaction
Repository: hbase Updated Branches: refs/heads/branch-1.4 dab7b39f2 -> d625b212e Updated Tags: refs/tags/1.4.9RC1 [created] c5b401c4f HBASE-21464 Splitting blocked with meta NSRE during split transaction Signed-off-by: Lars Hofhansl Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7c79c72 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7c79c72 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7c79c72 Branch: refs/heads/branch-1.4 Commit: e7c79c722376aa152dae6f976c01faebefed224f Parents: dab7b39 Author: Andrew Purtell Authored: Fri Nov 30 15:23:34 2018 -0800 Committer: Andrew Purtell Committed: Wed Dec 5 10:54:59 2018 -0800 -- .../hadoop/hbase/client/ConnectionManager.java | 42 1 file changed, 25 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e7c79c72/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java index 7cf09c2..35ffa3e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java @@ -1231,38 +1231,46 @@ class ConnectionManager { } } +private volatile RegionLocations metaLocations = null; +private volatile long lastMetaLookupTime = EnvironmentEdgeManager.currentTime(); +// cache meta location at most 10 seconds +private final static long META_LOOKUP_CACHE_INTERVAL = 1; + private RegionLocations locateMeta(final TableName tableName, boolean useCache, int replicaId) throws IOException { - // HBASE-10785: We cache the location of the META itself, so that we are not overloading - // zookeeper with one request for every region lookup. We cache the META with empty row - // key in MetaCache. - byte[] metaCacheKey = HConstants.EMPTY_START_ROW; // use byte[0] as the row for meta - RegionLocations locations = null; + // We cache the location of the META itself, so that we are not overloading + // zookeeper with one request for every region lookup. If relocating, bypass + // the cache immediately. if (useCache) { -locations = getCachedLocation(tableName, metaCacheKey); -if (locations != null && locations.getRegionLocation(replicaId) != null) { - return locations; +long now = EnvironmentEdgeManager.currentTime(); +if (now - lastMetaLookupTime < META_LOOKUP_CACHE_INTERVAL) { + if (metaLocations != null && + metaLocations.getRegionLocation(replicaId) != null) { +return metaLocations; + } +} else { + useCache = false; } } - // only one thread should do the lookup. synchronized (metaRegionLock) { // Check the cache again for a hit in case some other thread made the // same query while we were waiting on the lock. if (useCache) { - locations = getCachedLocation(tableName, metaCacheKey); - if (locations != null && locations.getRegionLocation(replicaId) != null) { -return locations; + if (metaLocations != null && + metaLocations.getRegionLocation(replicaId) != null) { +return metaLocations; } } - // Look up from zookeeper -locations = this.registry.getMetaRegionLocation(); -if (locations != null) { - cacheLocation(tableName, locations); +metaLocations = this.registry.getMetaRegionLocation(); +lastMetaLookupTime = EnvironmentEdgeManager.currentTime(); +if (metaLocations != null && +metaLocations.getRegionLocation(replicaId) != null) { + return metaLocations; } +return null; } - return locations; } /*
[1/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE during split transaction
Repository: hbase Updated Branches: refs/heads/branch-1 f1e2f077f -> bd87f4ebc refs/heads/branch-1.4 302edf447 -> 70d9934e4 HBASE-21464 Splitting blocked with meta NSRE during split transaction When looking up the locations of hbase:meta with useCache false, clear all previous cache entries for it first Fix Table reference leaks in MetaTableAccessor with try-with-resources Signed-off-by: Allan Yang Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bd87f4eb Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bd87f4eb Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bd87f4eb Branch: refs/heads/branch-1 Commit: bd87f4ebcd49e9a3d100bba81fd6ab8868027c06 Parents: f1e2f07 Author: Andrew Purtell Authored: Thu Nov 29 15:46:21 2018 -0800 Committer: Andrew Purtell Committed: Fri Nov 30 09:57:54 2018 -0800 -- .../apache/hadoop/hbase/MetaTableAccessor.java | 95 +--- .../hadoop/hbase/client/ConnectionManager.java | 10 ++- 2 files changed, 49 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/bd87f4eb/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 440f8c6..fd7a97b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -175,8 +175,7 @@ public class MetaTableAccessor { * @return An {@link Table} for hbase:meta * @throws IOException */ - static Table getMetaHTable(final Connection connection) - throws IOException { + static Table getMetaHTable(final Connection connection) throws IOException { // We used to pass whole CatalogTracker in here, now we just pass in Connection if (connection == null) { throw new NullPointerException("No connection"); @@ -248,11 +247,13 @@ public class MetaTableAccessor { } Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); -Result r = get(getMetaHTable(connection), get); -RegionLocations locations = getRegionLocations(r); -return locations == null - ? null - : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); +try (Table metaTable = getMetaHTable(connection)) { + Result r = get(metaTable, get); + RegionLocations locations = getRegionLocations(r); + return locations == null + ? null + : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); +} } /** @@ -267,8 +268,10 @@ public class MetaTableAccessor { byte[] row = getMetaKeyForRegion(regionInfo); Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); -Result r = get(getMetaHTable(connection), get); -return getRegionLocation(r, regionInfo, regionInfo.getReplicaId()); +try (Table metaTable = getMetaHTable(connection)) { + Result r = get(metaTable, get); + return getRegionLocation(r, regionInfo, regionInfo.getReplicaId()); +} } /** Returns the row key to use for this regionInfo */ @@ -300,7 +303,9 @@ public class MetaTableAccessor { byte[] regionName) throws IOException { Get get = new Get(regionName); get.addFamily(HConstants.CATALOG_FAMILY); -return get(getMetaHTable(connection), get); +try (Table metaTable = getMetaHTable(connection)) { + return get(metaTable, get); +} } /** @@ -631,19 +636,19 @@ public class MetaTableAccessor { scan.setCaching(caching); } scan.addFamily(HConstants.CATALOG_FAMILY); -Table metaTable = getMetaHTable(connection); -ResultScanner scanner = null; -try { - scanner = metaTable.getScanner(scan); - Result data; - while((data = scanner.next()) != null) { -if (data.isEmpty()) continue; -// Break if visit returns false. -if (!visitor.visit(data)) break; +try (Table metaTable = getMetaHTable(connection)) { + try (ResultScanner scanner = metaTable.getScanner(scan)) { +Result data; +while ((data = scanner.next()) != null) { + if (data.isEmpty()) { +continue; + } + // Break if visit returns false. + if (!visitor.visit(data)) { +break; + } +} } -} finally { - if (scanner != null) scanner.close(); - metaTable.close(); } } @@ -1020,7 +1025,9 @@ public class MetaTableAccessor { */ static void putToMetaTable(final Connection connection, final Put p) throws IOException { -
[2/2] hbase git commit: HBASE-21464 Splitting blocked with meta NSRE during split transaction
HBASE-21464 Splitting blocked with meta NSRE during split transaction When looking up the locations of hbase:meta with useCache false, clear all previous cache entries for it first Fix Table reference leaks in MetaTableAccessor with try-with-resources Signed-off-by: Allan Yang Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70d9934e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70d9934e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70d9934e Branch: refs/heads/branch-1.4 Commit: 70d9934e40785d13c9220d4b3ab0c3b31ca59f8c Parents: 302edf4 Author: Andrew Purtell Authored: Thu Nov 29 15:46:21 2018 -0800 Committer: Andrew Purtell Committed: Fri Nov 30 09:58:08 2018 -0800 -- .../apache/hadoop/hbase/MetaTableAccessor.java | 95 +--- .../hadoop/hbase/client/ConnectionManager.java | 10 ++- 2 files changed, 49 insertions(+), 56 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/70d9934e/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 440f8c6..fd7a97b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -175,8 +175,7 @@ public class MetaTableAccessor { * @return An {@link Table} for hbase:meta * @throws IOException */ - static Table getMetaHTable(final Connection connection) - throws IOException { + static Table getMetaHTable(final Connection connection) throws IOException { // We used to pass whole CatalogTracker in here, now we just pass in Connection if (connection == null) { throw new NullPointerException("No connection"); @@ -248,11 +247,13 @@ public class MetaTableAccessor { } Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); -Result r = get(getMetaHTable(connection), get); -RegionLocations locations = getRegionLocations(r); -return locations == null - ? null - : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); +try (Table metaTable = getMetaHTable(connection)) { + Result r = get(metaTable, get); + RegionLocations locations = getRegionLocations(r); + return locations == null + ? null + : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); +} } /** @@ -267,8 +268,10 @@ public class MetaTableAccessor { byte[] row = getMetaKeyForRegion(regionInfo); Get get = new Get(row); get.addFamily(HConstants.CATALOG_FAMILY); -Result r = get(getMetaHTable(connection), get); -return getRegionLocation(r, regionInfo, regionInfo.getReplicaId()); +try (Table metaTable = getMetaHTable(connection)) { + Result r = get(metaTable, get); + return getRegionLocation(r, regionInfo, regionInfo.getReplicaId()); +} } /** Returns the row key to use for this regionInfo */ @@ -300,7 +303,9 @@ public class MetaTableAccessor { byte[] regionName) throws IOException { Get get = new Get(regionName); get.addFamily(HConstants.CATALOG_FAMILY); -return get(getMetaHTable(connection), get); +try (Table metaTable = getMetaHTable(connection)) { + return get(metaTable, get); +} } /** @@ -631,19 +636,19 @@ public class MetaTableAccessor { scan.setCaching(caching); } scan.addFamily(HConstants.CATALOG_FAMILY); -Table metaTable = getMetaHTable(connection); -ResultScanner scanner = null; -try { - scanner = metaTable.getScanner(scan); - Result data; - while((data = scanner.next()) != null) { -if (data.isEmpty()) continue; -// Break if visit returns false. -if (!visitor.visit(data)) break; +try (Table metaTable = getMetaHTable(connection)) { + try (ResultScanner scanner = metaTable.getScanner(scan)) { +Result data; +while ((data = scanner.next()) != null) { + if (data.isEmpty()) { +continue; + } + // Break if visit returns false. + if (!visitor.visit(data)) { +break; + } +} } -} finally { - if (scanner != null) scanner.close(); - metaTable.close(); } } @@ -1020,7 +1025,9 @@ public class MetaTableAccessor { */ static void putToMetaTable(final Connection connection, final Put p) throws IOException { -put(getMetaHTable(connection), p); +try (Table metaTable = getMetaHTable(connection)) { + put(metaTable, p); +}