This is an automated email from the ASF dual-hosted git repository. openinx pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push: new 51b04f6 HBASE-21738 Remove all the CLSM#size operation in our memstore because it's an quite time consuming. 51b04f6 is described below commit 51b04f6f74e26215c92b57657fb41c474a0c5f02 Author: huzheng <open...@gmail.com> AuthorDate: Sat Jan 19 16:54:17 2019 +0800 HBASE-21738 Remove all the CLSM#size operation in our memstore because it's an quite time consuming. --- .../hbase/regionserver/AbstractMemStore.java | 4 +- .../hbase/regionserver/CSLMImmutableSegment.java | 4 +- .../regionserver/CellArrayImmutableSegment.java | 14 +++--- .../regionserver/CellChunkImmutableSegment.java | 26 +++++------ .../apache/hadoop/hbase/regionserver/CellSet.java | 3 ++ .../hbase/regionserver/CompactionPipeline.java | 50 +++++++++++++--------- .../regionserver/CompositeImmutableSegment.java | 2 +- .../hadoop/hbase/regionserver/DefaultMemStore.java | 4 +- .../apache/hadoop/hbase/regionserver/HRegion.java | 24 ++++++----- .../hadoop/hbase/regionserver/MemStoreSize.java | 26 +++++++---- .../hadoop/hbase/regionserver/MemStoreSizing.java | 27 ++++++++---- .../hadoop/hbase/regionserver/MutableSegment.java | 8 ++-- .../regionserver/NonThreadSafeMemStoreSizing.java | 21 ++++++--- .../regionserver/RegionServicesForStores.java | 5 ++- .../apache/hadoop/hbase/regionserver/Segment.java | 29 +++++++------ .../regionserver/ThreadSafeMemStoreSizing.java | 21 ++++++--- .../hbase/regionserver/TestCellSkipListSet.java | 8 ++-- .../hbase/regionserver/TestCompactingMemStore.java | 13 ++++-- .../TestCompactingToCellFlatMapMemStore.java | 6 +-- .../hadoop/hbase/regionserver/TestHStore.java | 8 ++-- .../regionserver/TestRegionServerAccounting.java | 31 +++++--------- 21 files changed, 194 insertions(+), 140 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index de3f114..b0b141c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -93,8 +93,8 @@ public abstract class AbstractMemStore implements MemStore { // regionServices can be null when testing if (regionServices != null) { regionServices.addMemStoreSize(memstoreAccounting.getDataSize(), - memstoreAccounting.getHeapSize(), - memstoreAccounting.getOffHeapSize()); + memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(), + memstoreAccounting.getCellsCount()); } this.timeOfOldestEdit = Long.MAX_VALUE; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java index 855fd08..52ff44a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java @@ -40,9 +40,9 @@ public class CSLMImmutableSegment extends ImmutableSegment { super(segment); // update the segment metadata heap size long indexOverhead = -MutableSegment.DEEP_OVERHEAD + DEEP_OVERHEAD_CSLM; - incMemStoreSize(0, indexOverhead, 0); // CSLM is always on-heap + incMemStoreSize(0, indexOverhead, 0, 0); // CSLM is always on-heap if (memstoreSizing != null) { - memstoreSizing.incMemStoreSize(0, indexOverhead, 0); + memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java index dadfc48..2608976 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java @@ -45,7 +45,7 @@ public class CellArrayImmutableSegment extends ImmutableSegment { protected CellArrayImmutableSegment(CellComparator comparator, MemStoreSegmentsIterator iterator, MemStoreLAB memStoreLAB, int numOfCells, MemStoreCompactionStrategy.Action action) { super(null, comparator, memStoreLAB); // initiailize the CellSet with NULL - incMemStoreSize(0, DEEP_OVERHEAD_CAM, 0); // CAM is always on-heap + incMemStoreSize(0, DEEP_OVERHEAD_CAM, 0, 0); // CAM is always on-heap // build the new CellSet based on CellArrayMap and update the CellSet of the new Segment initializeCellSet(numOfCells, iterator, action); } @@ -59,16 +59,18 @@ public class CellArrayImmutableSegment extends ImmutableSegment { MemStoreCompactionStrategy.Action action) { super(segment); // initiailize the upper class long indexOverhead = DEEP_OVERHEAD_CAM - CSLMImmutableSegment.DEEP_OVERHEAD_CSLM; - incMemStoreSize(0, indexOverhead, 0); // CAM is always on-heap + incMemStoreSize(0, indexOverhead, 0, 0); // CAM is always on-heap + memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0); int numOfCells = segment.getCellsCount(); // build the new CellSet based on CellChunkMap and update the CellSet of this Segment reinitializeCellSet(numOfCells, segment.getScanner(Long.MAX_VALUE), segment.getCellSet(), - action); + action); // arrange the meta-data size, decrease all meta-data sizes related to SkipList; // add sizes of CellArrayMap entry (reinitializeCellSet doesn't take the care for the sizes) - long newSegmentSizeDelta = numOfCells*(indexEntrySize()-ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); - incMemStoreSize(0, newSegmentSizeDelta, 0); - memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0); + long newSegmentSizeDelta = + numOfCells * (indexEntrySize() - ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); + incMemStoreSize(0, newSegmentSizeDelta, 0, 0); + memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0, 0); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java index e2f8205..9653e75 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java @@ -57,10 +57,10 @@ public class CellChunkImmutableSegment extends ImmutableSegment { // memStoreLAB cannot be null in this class boolean onHeap = getMemStoreLAB().isOnHeap(); // initiate the heapSize with the size of the segment metadata - if(onHeap) { - incMemStoreSize(0, indexOverhead, 0); + if (onHeap) { + incMemStoreSize(0, indexOverhead, 0, 0); } else { - incMemStoreSize(0, 0, indexOverhead); + incMemStoreSize(0, 0, indexOverhead, 0); } // build the new CellSet based on CellArrayMap and update the CellSet of the new Segment initializeCellSet(numOfCells, iterator, action); @@ -79,9 +79,12 @@ public class CellChunkImmutableSegment extends ImmutableSegment { boolean onHeap = getMemStoreLAB().isOnHeap(); // initiate the heapSize with the size of the segment metadata if(onHeap) { - incMemStoreSize(0, indexOverhead, 0); + incMemStoreSize(0, indexOverhead, 0, 0); + memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0); } else { - incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM); + incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM, 0); + memstoreSizing.incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM, + 0); } int numOfCells = segment.getCellsCount(); // build the new CellSet based on CellChunkMap @@ -92,12 +95,11 @@ public class CellChunkImmutableSegment extends ImmutableSegment { // (reinitializeCellSet doesn't take the care for the sizes) long newSegmentSizeDelta = numOfCells*(indexEntrySize()-ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); if(onHeap) { - incMemStoreSize(0, newSegmentSizeDelta, 0); - memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0); + incMemStoreSize(0, newSegmentSizeDelta, 0, 0); + memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0, 0); } else { - incMemStoreSize(0, 0, newSegmentSizeDelta); - memstoreSizing.incMemStoreSize(0, 0, newSegmentSizeDelta); - + incMemStoreSize(0, 0, newSegmentSizeDelta, 0); + memstoreSizing.incMemStoreSize(0, 0, newSegmentSizeDelta, 0); } } @@ -332,8 +334,8 @@ public class CellChunkImmutableSegment extends ImmutableSegment { long newCellSize = getCellLength(cell); long heapOverhead = newHeapSize - oldHeapSize; long offHeapOverhead = newOffHeapSize - oldOffHeapSize; - //TODO: maybe need to update the dataSize of the region - incMemStoreSize(newCellSize - oldCellSize, heapOverhead, offHeapOverhead); + // TODO: maybe need to update the dataSize of the region + incMemStoreSize(newCellSize - oldCellSize, heapOverhead, offHeapOverhead, 0); return cell; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java index 94a256d..5190d7b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java @@ -217,6 +217,9 @@ public class CellSet implements NavigableSet<Cell> { @Override public int size() { + if (delegatee instanceof ConcurrentSkipListMap) { + throw new UnsupportedOperationException("ConcurrentSkipListMap.size() is time-consuming"); + } return this.delegatee.size(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java index 5afaec1..cdec051 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java @@ -78,9 +78,8 @@ public class CompactionPipeline { ImmutableSegment immutableSegment = SegmentFactory.instance(). createImmutableSegment(segment, memstoreAccounting); if (region != null) { - region.addMemStoreSize(memstoreAccounting.getDataSize(), - memstoreAccounting.getHeapSize(), - memstoreAccounting.getOffHeapSize()); + region.addMemStoreSize(memstoreAccounting.getDataSize(), memstoreAccounting.getHeapSize(), + memstoreAccounting.getOffHeapSize(), memstoreAccounting.getCellsCount()); } synchronized (pipeline){ boolean res = addFirst(immutableSegment); @@ -142,31 +141,31 @@ public class CompactionPipeline { if (updateRegionSize && region != null) { // update the global memstore size counter long suffixDataSize = getSegmentsKeySize(suffix); - long newDataSize = 0; - if(segment != null) { - newDataSize = segment.getDataSize(); - } - long dataSizeDelta = suffixDataSize - newDataSize; long suffixHeapSize = getSegmentsHeapSize(suffix); long suffixOffHeapSize = getSegmentsOffHeapSize(suffix); + int suffixCellsCount = getSegmentsCellsCount(suffix); + long newDataSize = 0; long newHeapSize = 0; long newOffHeapSize = 0; - if(segment != null) { + int newCellsCount = 0; + if (segment != null) { + newDataSize = segment.getDataSize(); newHeapSize = segment.getHeapSize(); newOffHeapSize = segment.getOffHeapSize(); + newCellsCount = segment.getCellsCount(); } + long dataSizeDelta = suffixDataSize - newDataSize; long offHeapSizeDelta = suffixOffHeapSize - newOffHeapSize; long heapSizeDelta = suffixHeapSize - newHeapSize; - region.addMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta); - LOG.debug("Suffix data size={}, new segment data size={}, " - + "suffix heap size={}," + "new segment heap size={}" - + "suffix off heap size={}," + "new segment off heap size={}" - , suffixDataSize - , newDataSize - , suffixHeapSize - , newHeapSize - , suffixOffHeapSize - , newOffHeapSize); + int cellsCountDelta = suffixCellsCount - newCellsCount; + region.addMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta, -cellsCountDelta); + LOG.debug( + "Suffix data size={}, new segment data size={}, " + + "suffix heap size={}, new segment heap size={}" + + "suffix off heap size={}, new segment off heap size={}" + + "suffix cells count={}, new cells count={}", + suffixDataSize, newDataSize, suffixHeapSize, newHeapSize, suffixOffHeapSize, newOffHeapSize, + suffixCellsCount, newCellsCount); } return true; } @@ -195,6 +194,14 @@ public class CompactionPipeline { return res; } + private static int getSegmentsCellsCount(List<? extends Segment> list) { + int res = 0; + for (Segment segment : list) { + res += segment.getCellsCount(); + } + return res; + } + /** * If the caller holds the current version, go over the the pipeline and try to flatten each * segment. Flattening is replacing the ConcurrentSkipListMap based CellSet to CellArrayMap based. @@ -226,12 +233,13 @@ public class CompactionPipeline { ImmutableSegment newS = SegmentFactory.instance().createImmutableSegmentByFlattening( (CSLMImmutableSegment)s,idxType,newMemstoreAccounting,action); replaceAtIndex(i,newS); - if(region != null) { + if (region != null) { // Update the global memstore size counter upon flattening there is no change in the // data size MemStoreSize mss = newMemstoreAccounting.getMemStoreSize(); Preconditions.checkArgument(mss.getDataSize() == 0, "Not zero!"); - region.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + region.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), + mss.getCellsCount()); } LOG.debug("Compaction pipeline segment {} flattened", s); return true; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java index dcfaf81..3dcfa55 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java @@ -190,7 +190,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * Updates the heap size counter of the segment by the given delta */ @Override - public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead) { + public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index bd6c8ac..a4f14d8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -104,8 +104,8 @@ public class DefaultMemStore extends AbstractMemStore { // regionServices can be null when testing if (regionServices != null) { regionServices.addMemStoreSize(memstoreAccounting.getDataSize(), - memstoreAccounting.getHeapSize(), - memstoreAccounting.getOffHeapSize()); + memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(), + memstoreAccounting.getCellsCount()); } this.snapshot = immutableSegment; resetActive(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index afa7b84..3da0bcd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1238,28 +1238,32 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * store */ void incMemStoreSize(MemStoreSize mss) { - incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), + mss.getCellsCount()); } - void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) { + void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, + int cellsCountDelta) { if (this.rsAccounting != null) { rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); } - long dataSize = - this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); + long dataSize = this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, + offHeapSizeDelta, cellsCountDelta); checkNegativeMemStoreDataSize(dataSize, dataSizeDelta); } void decrMemStoreSize(MemStoreSize mss) { - decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), + mss.getCellsCount()); } - void decrMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) { + void decrMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, + int cellsCountDelta) { if (this.rsAccounting != null) { rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); } - long dataSize = - this.memStoreSizing.decMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); + long dataSize = this.memStoreSizing.decMemStoreSize(dataSizeDelta, heapSizeDelta, + offHeapSizeDelta, cellsCountDelta); checkNegativeMemStoreDataSize(dataSize, -dataSizeDelta); } @@ -2737,7 +2741,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // Set down the memstore size by amount of flush. MemStoreSize mss = prepareResult.totalFlushableSize.getMemStoreSize(); - this.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + this.decrMemStoreSize(mss); if (wal != null) { // write flush marker to WAL. If fail, we should throw DroppedSnapshotException @@ -3153,7 +3157,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi }); // update memStore size region.incMemStoreSize(memStoreAccounting.getDataSize(), memStoreAccounting.getHeapSize(), - memStoreAccounting.getOffHeapSize()); + memStoreAccounting.getOffHeapSize(), memStoreAccounting.getCellsCount()); } public boolean isDone() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java index 97a416e..86cf810 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java @@ -44,20 +44,23 @@ public class MemStoreSize { */ private final long offHeapSize; + private final int cellsCount; + /** * Package private constructor. */ MemStoreSize() { - this(0L, 0L, 0L); + this(0L, 0L, 0L, 0); } /** * Package private constructor. */ - MemStoreSize(long dataSize, long heapSize, long offHeapSize) { + MemStoreSize(long dataSize, long heapSize, long offHeapSize, int cellsCount) { this.dataSize = dataSize; this.heapSize = heapSize; this.offHeapSize = offHeapSize; + this.cellsCount = cellsCount; } /** @@ -67,10 +70,12 @@ public class MemStoreSize { this.dataSize = memStoreSize.getDataSize(); this.heapSize = memStoreSize.getHeapSize(); this.offHeapSize = memStoreSize.getOffHeapSize(); + this.cellsCount = memStoreSize.getCellsCount(); } public boolean isEmpty() { - return this.dataSize == 0 && this.heapSize == 0 && this.offHeapSize == 0; + return this.dataSize == 0 && this.heapSize == 0 && this.offHeapSize == 0 + && this.cellsCount == 0; } public long getDataSize() { @@ -85,6 +90,10 @@ public class MemStoreSize { return this.offHeapSize; } + public int getCellsCount() { + return this.cellsCount; + } + @Override public boolean equals(Object obj) { if (obj == null) { @@ -93,9 +102,9 @@ public class MemStoreSize { if (!(obj instanceof MemStoreSize)) { return false; } - MemStoreSize other = (MemStoreSize)obj; - return this.dataSize == other.dataSize && this.heapSize == other.heapSize && - this.offHeapSize == other.offHeapSize; + MemStoreSize other = (MemStoreSize) obj; + return this.dataSize == other.dataSize && this.heapSize == other.heapSize + && this.offHeapSize == other.offHeapSize && this.cellsCount == other.cellsCount; } @Override @@ -103,12 +112,13 @@ public class MemStoreSize { long h = 31 * this.dataSize; h = h + 31 * this.heapSize; h = h + 31 * this.offHeapSize; + h = h + 31 * this.cellsCount; return (int) h; } @Override public String toString() { - return "dataSize=" + this.dataSize + ", getHeapSize=" + this.heapSize + - ", getOffHeapSize=" + this.offHeapSize; + return "dataSize=" + this.dataSize + ", getHeapSize=" + this.heapSize + ", getOffHeapSize=" + + this.offHeapSize + ", getCellsCount=" + this.cellsCount; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java index 8430ac6..87297f6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java @@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience; */ @InterfaceAudience.Private public interface MemStoreSizing { - static final MemStoreSizing DUD = new MemStoreSizing() { + MemStoreSizing DUD = new MemStoreSizing() { private final MemStoreSize mss = new MemStoreSize(); @Override @@ -77,8 +77,13 @@ public interface MemStoreSizing { } @Override - public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, - long offHeapSizeDelta) { + public int getCellsCount() { + return this.mss.getCellsCount(); + } + + @Override + public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, + int cellsCountDelta) { throw new RuntimeException("I'm a DUD, you can't use me!"); } }; @@ -86,31 +91,35 @@ public interface MemStoreSizing { /** * @return The new dataSize ONLY as a convenience */ - long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta); + long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, + int cellsCount); default long incMemStoreSize(MemStoreSize delta) { - return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize()); + return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize(), + delta.getCellsCount()); } /** * @return The new dataSize ONLY as a convenience */ default long decMemStoreSize(long dataSizeDelta, long heapSizeDelta, - long offHeapSizeDelta) { - return incMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta); + long offHeapSizeDelta, int cellsCountDelta) { + return incMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta, -cellsCountDelta); } default long decMemStoreSize(MemStoreSize delta) { - return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize()); + return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize(), + -delta.getCellsCount()); } long getDataSize(); long getHeapSize(); long getOffHeapSize(); + int getCellsCount(); /** * @return Use this datastructure to return all three settings, {@link #getDataSize()}, * {@link #getHeapSize()}, and {@link #getOffHeapSize()}, in the one go. */ MemStoreSize getMemStoreSize(); -} \ No newline at end of file +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java index 714e9bc..a5cdda9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java @@ -45,9 +45,9 @@ public class MutableSegment extends Segment { protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, MemStoreSizing memstoreSizing) { super(cellSet, comparator, memStoreLAB, TimeRangeTracker.create(TimeRangeTracker.Type.SYNC)); - incMemStoreSize(0, DEEP_OVERHEAD, 0); // update the mutable segment metadata + incMemStoreSize(0, DEEP_OVERHEAD, 0, 0); // update the mutable segment metadata if (memstoreSizing != null) { - memstoreSizing.incMemStoreSize(0, DEEP_OVERHEAD, 0); + memstoreSizing.incMemStoreSize(0, DEEP_OVERHEAD, 0, 0); } } @@ -93,9 +93,9 @@ public class MutableSegment extends Segment { int cellLen = getCellLength(cur); long heapSize = heapSizeChange(cur, true); long offHeapSize = offHeapSizeChange(cur, true); - incMemStoreSize(-cellLen, -heapSize, -offHeapSize); + incMemStoreSize(-cellLen, -heapSize, -offHeapSize, -1); if (memStoreSizing != null) { - memStoreSizing.decMemStoreSize(cellLen, heapSize, offHeapSize); + memStoreSizing.decMemStoreSize(cellLen, heapSize, offHeapSize, 1); } it.remove(); } else { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java index 601ff33..ba5db0a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java @@ -32,30 +32,32 @@ class NonThreadSafeMemStoreSizing implements MemStoreSizing { private long dataSize = 0; private long heapSize = 0; private long offHeapSize = 0; + private int cellsCount = 0; NonThreadSafeMemStoreSizing() { - this(0, 0, 0); + this(0, 0, 0, 0); } NonThreadSafeMemStoreSizing(MemStoreSize mss) { - this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); } - NonThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize) { - incMemStoreSize(dataSize, heapSize, offHeapSize); + NonThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize, int cellsCount) { + incMemStoreSize(dataSize, heapSize, offHeapSize, cellsCount); } @Override public MemStoreSize getMemStoreSize() { - return new MemStoreSize(this.dataSize, this.heapSize, this.offHeapSize); + return new MemStoreSize(this.dataSize, this.heapSize, this.offHeapSize, this.cellsCount); } @Override public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, - long offHeapSizeDelta) { + long offHeapSizeDelta, int cellsCountDelta) { this.offHeapSize += offHeapSizeDelta; this.heapSize += heapSizeDelta; this.dataSize += dataSizeDelta; + this.cellsCount += cellsCountDelta; return this.dataSize; } @@ -75,7 +77,12 @@ class NonThreadSafeMemStoreSizing implements MemStoreSizing { } @Override + public int getCellsCount() { + return cellsCount; + } + + @Override public String toString() { return getMemStoreSize().toString(); } -} \ No newline at end of file +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java index b088856..c1af9db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java @@ -65,8 +65,9 @@ public class RegionServicesForStores { region.unblockUpdates(); } - public void addMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) { - region.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta); + public void addMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, + int cellsCountDelta) { + region.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta, cellsCountDelta); } public RegionInfo getRegionInfo() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index 1f414a7..bf2f9de 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -80,21 +80,24 @@ public abstract class Segment implements MemStoreSizing { long dataSize = 0; long heapSize = 0; long OffHeapSize = 0; + int cellsCount = 0; for (Segment segment : segments) { MemStoreSize memStoreSize = segment.getMemStoreSize(); dataSize += memStoreSize.getDataSize(); heapSize += memStoreSize.getHeapSize(); OffHeapSize += memStoreSize.getOffHeapSize(); + cellsCount += memStoreSize.getCellsCount(); } this.comparator = comparator; // Do we need to be thread safe always? What if ImmutableSegment? // DITTO for the TimeRangeTracker below. - this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize); + this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount); this.timeRangeTracker = trt; } // This constructor is used to create empty Segments. - protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, TimeRangeTracker trt) { + protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, + TimeRangeTracker trt) { this.cellSet.set(cellSet); this.comparator = comparator; this.minSequenceId = Long.MAX_VALUE; @@ -135,12 +138,6 @@ public abstract class Segment implements MemStoreSizing { return getCellSet().isEmpty(); } - /** - * @return number of cells in segment - */ - public int getCellsCount() { - return getCellSet().size(); - } /** * Closing a segment before it is being discarded @@ -169,7 +166,7 @@ public abstract class Segment implements MemStoreSizing { return cell; } - Cell cellFromMslab = null; + Cell cellFromMslab; if (forceCloneOfBigCell) { cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell); } else { @@ -240,8 +237,13 @@ public abstract class Segment implements MemStoreSizing { } @Override - public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead) { - return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead); + public int getCellsCount() { + return this.memStoreSizing.getCellsCount(); + } + + @Override + public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) { + return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount); } public long getMinSequenceId() { @@ -296,6 +298,7 @@ public abstract class Segment implements MemStoreSizing { protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, MemStoreSizing memstoreSizing) { long cellSize = 0; + int cellsCount = succ ? 1 : 0; // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger // than the counted number) @@ -308,9 +311,9 @@ public abstract class Segment implements MemStoreSizing { // is needed. long heapSize = heapSizeChange(cellToAdd, sizeChanged); long offHeapSize = offHeapSizeChange(cellToAdd, sizeChanged); - incMemStoreSize(cellSize, heapSize, offHeapSize); + incMemStoreSize(cellSize, heapSize, offHeapSize, cellsCount); if (memstoreSizing != null) { - memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize); + memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize, cellsCount); } getTimeRangeTracker().includeTimestamp(cellToAdd); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java index de05493..3dec1d6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.yetus.audience.InterfaceAudience; @@ -35,26 +36,29 @@ class ThreadSafeMemStoreSizing implements MemStoreSizing { private final AtomicLong dataSize = new AtomicLong(); private final AtomicLong heapSize = new AtomicLong(); private final AtomicLong offHeapSize = new AtomicLong(); + private final AtomicInteger cellsCount = new AtomicInteger(); ThreadSafeMemStoreSizing() { - this(0, 0, 0); + this(0, 0, 0, 0); } ThreadSafeMemStoreSizing(MemStoreSize mss) { - this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount()); } - ThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize) { - incMemStoreSize(dataSize, heapSize, offHeapSize); + ThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize, int cellsCount) { + incMemStoreSize(dataSize, heapSize, offHeapSize, cellsCount); } public MemStoreSize getMemStoreSize() { - return new MemStoreSize(getDataSize(), getHeapSize(), getOffHeapSize()); + return new MemStoreSize(getDataSize(), getHeapSize(), getOffHeapSize(), getCellsCount()); } - public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) { + public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta, + int cellsCount) { this.offHeapSize.addAndGet(offHeapSizeDelta); this.heapSize.addAndGet(heapSizeDelta); + this.cellsCount.addAndGet(cellsCount); return this.dataSize.addAndGet(dataSizeDelta); } @@ -74,6 +78,11 @@ class ThreadSafeMemStoreSizing implements MemStoreSizing { } @Override + public int getCellsCount() { + return cellsCount.get(); + } + + @Override public String toString() { return getMemStoreSize().toString(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java index 4f46bcf..c2e2ca1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java @@ -48,20 +48,20 @@ public class TestCellSkipListSet extends TestCase { } public void testAdd() throws Exception { - byte [] bytes = Bytes.toBytes(getName()); + byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); this.csls.add(kv); assertTrue(this.csls.contains(kv)); - assertEquals(1, this.csls.size()); + assertEquals(1, this.csls.getDelegatee().size()); Cell first = this.csls.first(); assertTrue(kv.equals(first)); assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(), first.getValueArray(), first.getValueOffset(), first.getValueLength())); // Now try overwritting - byte [] overwriteValue = Bytes.toBytes("overwrite"); + byte[] overwriteValue = Bytes.toBytes("overwrite"); KeyValue overwrite = new KeyValue(bytes, bytes, bytes, overwriteValue); this.csls.add(overwrite); - assertEquals(1, this.csls.size()); + assertEquals(1, this.csls.getDelegatee().size()); first = this.csls.first(); assertTrue(Bytes.equals(overwrite.getValueArray(), overwrite.getValueOffset(), overwrite.getValueLength(), first.getValueArray(), first.getValueOffset(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 8dbddb9..eef91d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -679,7 +679,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore { mss = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot // simulate flusher - region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), + mss.getCellsCount()); ImmutableSegment s = memstore.getSnapshot(); assertEquals(7, s.getCellsCount()); assertEquals(0, regionServicesForStores.getMemStoreSize()); @@ -756,7 +757,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore { mss = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot // simulate flusher - region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), + mss.getCellsCount()); ImmutableSegment s = memstore.getSnapshot(); assertEquals(4, s.getCellsCount()); assertEquals(0, regionServicesForStores.getMemStoreSize()); @@ -831,6 +833,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); + int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); @@ -843,7 +846,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore { LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, - hmc.getActive().getHeapSize() - heapOverhead, 0); + hmc.getActive().getHeapSize() - heapOverhead, 0, + hmc.getActive().getCellsCount() - cellsCount); return totalLen; } @@ -853,6 +857,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); + int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); @@ -864,7 +869,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, - hmc.getActive().getHeapSize() - heapOverhead, 0); + hmc.getActive().getHeapSize() - heapOverhead, 0, cellsCount); return totalLen; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index 70d6970..9ba3613 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -283,13 +283,13 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore mss = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot // simulate flusher - region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize()); + region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), + mss.getCellsCount()); ImmutableSegment s = memstore.getSnapshot(); assertEquals(4, s.getCellsCount()); assertEquals(0, regionServicesForStores.getMemStoreSize()); memstore.clearSnapshot(snapshot.getId()); - } ////////////////////////////////////////////////////////////////////////////// @@ -914,7 +914,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore } MemStoreSize mss = memstoreSizing.getMemStoreSize(); regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), - mss.getOffHeapSize()); + mss.getOffHeapSize(), mss.getCellsCount()); return mss.getDataSize(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java index 1b35b95..46ba464 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java @@ -264,7 +264,7 @@ public class TestHStore { MemStoreSizing kvSize = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); // add the heap size of active (mutable) segment - kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0); + kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0); mss = store.memstore.getFlushableSize(); assertEquals(kvSize.getMemStoreSize(), mss); // Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right. @@ -277,12 +277,12 @@ public class TestHStore { } // due to snapshot, change mutable to immutable segment kvSize.incMemStoreSize(0, - CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD, 0); + CSLMImmutableSegment.DEEP_OVERHEAD_CSLM - MutableSegment.DEEP_OVERHEAD, 0, 0); mss = store.memstore.getFlushableSize(); assertEquals(kvSize.getMemStoreSize(), mss); MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing(); - store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2); - kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0); + store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2); + kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0); // Even though we add a new kv, we expect the flushable size to be 'same' since we have // not yet cleared the snapshot -- the above flush failed. assertEquals(kvSize.getMemStoreSize(), mss); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java index fb246d5..bf4bca0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java @@ -50,22 +50,18 @@ public class TestRegionServerAccounting { public void testOnheapMemstoreHigherWaterMarkLimits() { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); long dataSize = regionServerAccounting.getGlobalMemStoreLimit(); - MemStoreSize memstoreSize = - new MemStoreSize(dataSize, dataSize, 0); + MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 0); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); - assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, - regionServerAccounting.isAboveHighWaterMark()); + assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark()); } @Test public void testOnheapMemstoreLowerWaterMarkLimits() { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); long dataSize = regionServerAccounting.getGlobalMemStoreLimit(); - MemStoreSize memstoreSize = - new MemStoreSize(dataSize, dataSize, 0); + MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 0); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); - assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, - regionServerAccounting.isAboveLowWaterMark()); + assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark()); } @Test @@ -76,7 +72,7 @@ public class TestRegionServerAccounting { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach offheap limit as data size is higher and not due to heap size MemStoreSize memstoreSize = - new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L)); + new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L), 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); assertEquals(FlushType.ABOVE_OFFHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark()); @@ -90,11 +86,9 @@ public class TestRegionServerAccounting { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach higher limit as heap size is higher and not due to offheap size long dataSize = regionServerAccounting.getGlobalOnHeapMemStoreLimit(); - MemStoreSize memstoreSize = - new MemStoreSize(dataSize, dataSize, 0); + MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); - assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, - regionServerAccounting.isAboveHighWaterMark()); + assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark()); } @Test @@ -105,10 +99,9 @@ public class TestRegionServerAccounting { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach offheap limit as data size is higher and not due to heap size MemStoreSize memstoreSize = - new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L)); + new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L), 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); - assertEquals(FlushType.ABOVE_OFFHEAP_LOWER_MARK, - regionServerAccounting.isAboveLowWaterMark()); + assertEquals(FlushType.ABOVE_OFFHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark()); } @Test @@ -119,10 +112,8 @@ public class TestRegionServerAccounting { RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf); // this will breach higher limit as heap size is higher and not due to offheap size long dataSize = regionServerAccounting.getGlobalOnHeapMemStoreLimit(); - MemStoreSize memstoreSize = - new MemStoreSize(dataSize, dataSize, 0); + MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 100); regionServerAccounting.incGlobalMemStoreSize(memstoreSize); - assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, - regionServerAccounting.isAboveLowWaterMark()); + assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark()); } }