Repository: hbase Updated Branches: refs/heads/master 67f013430 -> df5de33a0
HBASE-20196 Maintain all regions with same size in memstore flusher Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df5de33a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df5de33a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df5de33a Branch: refs/heads/master Commit: df5de33a022ddf7acc8adc33d9616a742fcaaea0 Parents: 67f0134 Author: tedyu <[email protected]> Authored: Mon Mar 19 08:10:39 2018 -0700 Committer: tedyu <[email protected]> Committed: Mon Mar 19 08:10:39 2018 -0700 ---------------------------------------------------------------------- .../hbase/regionserver/HRegionServer.java | 25 ++++++----- .../hbase/regionserver/MemStoreFlusher.java | 47 +++++++++++--------- 2 files changed, 41 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/df5de33a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 70d9116..6520959 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -2759,14 +2759,20 @@ public class HRegionServer extends HasThread implements configurationManager.registerObserver(region); } + private void addRegion(SortedMap<Long, Collection<HRegion>> sortedRegions, HRegion region, + long size) { + if (!sortedRegions.containsKey(size)) { + sortedRegions.put(size, new ArrayList<>()); + } + sortedRegions.get(size).add(region); + } /** * @return A new Map of online regions sorted by region off-heap size with the first entry being - * the biggest. If two regions are the same size, then the last one found wins; i.e. this - * method may NOT return all regions. + * the biggest. */ - SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedByOffHeapSize() { + SortedMap<Long, Collection<HRegion>> getCopyOfOnlineRegionsSortedByOffHeapSize() { // we'll sort the regions in reverse - SortedMap<Long, HRegion> sortedRegions = new TreeMap<>( + SortedMap<Long, Collection<HRegion>> sortedRegions = new TreeMap<>( new Comparator<Long>() { @Override public int compare(Long a, Long b) { @@ -2775,19 +2781,18 @@ public class HRegionServer extends HasThread implements }); // Copy over all regions. Regions are sorted by size with biggest first. for (HRegion region : this.onlineRegions.values()) { - sortedRegions.put(region.getMemStoreOffHeapSize(), region); + addRegion(sortedRegions, region, region.getMemStoreOffHeapSize()); } return sortedRegions; } /** * @return A new Map of online regions sorted by region heap size with the first entry being the - * biggest. If two regions are the same size, then the last one found wins; i.e. this method - * may NOT return all regions. + * biggest. */ - SortedMap<Long, HRegion> getCopyOfOnlineRegionsSortedByOnHeapSize() { + SortedMap<Long, Collection<HRegion>> getCopyOfOnlineRegionsSortedByOnHeapSize() { // we'll sort the regions in reverse - SortedMap<Long, HRegion> sortedRegions = new TreeMap<>( + SortedMap<Long, Collection<HRegion>> sortedRegions = new TreeMap<>( new Comparator<Long>() { @Override public int compare(Long a, Long b) { @@ -2796,7 +2801,7 @@ public class HRegionServer extends HasThread implements }); // Copy over all regions. Regions are sorted by size with biggest first. for (HRegion region : this.onlineRegions.values()) { - sortedRegions.put(region.getMemStoreHeapSize(), region); + addRegion(sortedRegions, region, region.getMemStoreHeapSize()); } return sortedRegions; } http://git-wip-us.apache.org/repos/asf/hbase/blob/df5de33a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index 23321e8..cef7256 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.util.ArrayList; +import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.HashMap; import java.util.HashSet; @@ -155,7 +156,7 @@ class MemStoreFlusher implements FlushRequester { * @return true if successful */ private boolean flushOneForGlobalPressure() { - SortedMap<Long, HRegion> regionsBySize = null; + SortedMap<Long, Collection<HRegion>> regionsBySize = null; switch(flushType) { case ABOVE_OFFHEAP_HIGHER_MARK: case ABOVE_OFFHEAP_LOWER_MARK: @@ -387,41 +388,45 @@ class MemStoreFlusher implements FlushRequester { } private HRegion getBiggestMemStoreRegion( - SortedMap<Long, HRegion> regionsBySize, + SortedMap<Long, Collection<HRegion>> regionsBySize, Set<HRegion> excludedRegions, boolean checkStoreFileCount) { synchronized (regionsInQueue) { - for (HRegion region : regionsBySize.values()) { - if (excludedRegions.contains(region)) { - continue; - } + for (Map.Entry<Long, Collection<HRegion>> entry : regionsBySize.entrySet()) { + for (HRegion region : entry.getValue()) { + if (excludedRegions.contains(region)) { + continue; + } - if (region.writestate.flushing || !region.writestate.writesEnabled) { - continue; - } + if (region.writestate.flushing || !region.writestate.writesEnabled) { + continue; + } - if (checkStoreFileCount && isTooManyStoreFiles(region)) { - continue; + if (checkStoreFileCount && isTooManyStoreFiles(region)) { + continue; + } + return region; } - return region; } } return null; } - private HRegion getBiggestMemStoreOfRegionReplica(SortedMap<Long, HRegion> regionsBySize, + private HRegion getBiggestMemStoreOfRegionReplica( + SortedMap<Long, Collection<HRegion>> regionsBySize, Set<HRegion> excludedRegions) { synchronized (regionsInQueue) { - for (HRegion region : regionsBySize.values()) { - if (excludedRegions.contains(region)) { - continue; - } + for (Map.Entry<Long, Collection<HRegion>> entry : regionsBySize.entrySet()) { + for (HRegion region : entry.getValue()) { + if (excludedRegions.contains(region)) { + continue; + } - if (RegionReplicaUtil.isDefaultReplica(region.getRegionInfo())) { - continue; + if (RegionReplicaUtil.isDefaultReplica(region.getRegionInfo())) { + continue; + } + return region; } - - return region; } } return null;
