HBASE-16229 Cleaning up size and heapSize calculation.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77b32732 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77b32732 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77b32732 Branch: refs/heads/hbase-12439 Commit: 77b327320a72ca01b35f655c42f8c13f659dff31 Parents: 2566cfe Author: anoopsamjohn <anoopsamj...@gmail.com> Authored: Tue Sep 13 11:43:26 2016 +0530 Committer: Gary Helmling <ga...@apache.org> Committed: Mon Sep 12 23:23:38 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/util/ClassSize.java | 15 +++ .../hbase/regionserver/AbstractMemStore.java | 49 +++---- .../hbase/regionserver/CompactingMemStore.java | 92 ++++++------- .../hbase/regionserver/CompactionPipeline.java | 31 +++-- .../hbase/regionserver/DefaultMemStore.java | 30 ++--- .../hbase/regionserver/ImmutableSegment.java | 84 ++++++------ .../hadoop/hbase/regionserver/MemStore.java | 6 +- .../hbase/regionserver/MemStoreCompactor.java | 24 ++-- .../hbase/regionserver/MemStoreSnapshot.java | 2 +- .../hbase/regionserver/MutableSegment.java | 25 ++-- .../hadoop/hbase/regionserver/Segment.java | 103 ++++++--------- .../hbase/regionserver/SegmentFactory.java | 43 +++---- .../apache/hadoop/hbase/io/TestHeapSize.java | 92 ++++++++++++- .../regionserver/TestCompactingMemStore.java | 22 +--- .../TestCompactingToCellArrayMapMemStore.java | 4 +- .../regionserver/TestPerColumnFamilyFlush.java | 49 +++---- .../TestWalAndCompactingMemStoreFlush.java | 129 +++++++------------ 17 files changed, 417 insertions(+), 383 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java index ff9dbcb..85a6483 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -46,6 +46,12 @@ public class ClassSize { /** Overhead for ArrayList(0) */ public static final int ARRAYLIST; + /** Overhead for LinkedList(0) */ + public static final int LINKEDLIST; + + /** Overhead for a single entry in LinkedList */ + public static final int LINKEDLIST_ENTRY; + /** Overhead for ByteBuffer */ public static final int BYTE_BUFFER; @@ -100,6 +106,9 @@ public class ClassSize { /** Overhead for AtomicBoolean */ public static final int ATOMIC_BOOLEAN; + /** Overhead for AtomicReference */ + public static final int ATOMIC_REFERENCE; + /** Overhead for CopyOnWriteArraySet */ public static final int COPYONWRITE_ARRAYSET; @@ -240,6 +249,10 @@ public class ClassSize { ARRAYLIST = align(OBJECT + REFERENCE + (2 * Bytes.SIZEOF_INT)) + align(ARRAY); + LINKEDLIST = align(OBJECT + (2 * Bytes.SIZEOF_INT) + (2 * REFERENCE)); + + LINKEDLIST_ENTRY = align(OBJECT + (2 * REFERENCE)); + //noinspection PointlessArithmeticExpression BYTE_BUFFER = align(OBJECT + REFERENCE + (5 * Bytes.SIZEOF_INT) + @@ -292,6 +305,8 @@ public class ClassSize { ATOMIC_BOOLEAN = align(OBJECT + Bytes.SIZEOF_BOOLEAN); + ATOMIC_REFERENCE = align(OBJECT + REFERENCE); + COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE); COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY); http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 419b76a..5e9f632 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -52,34 +52,29 @@ public abstract class AbstractMemStore implements MemStore { private final CellComparator comparator; // active segment absorbs write operations - private volatile MutableSegment active; + protected volatile MutableSegment active; // Snapshot of memstore. Made for flusher. - private volatile ImmutableSegment snapshot; + protected volatile ImmutableSegment snapshot; protected volatile long snapshotId; // Used to track when to flush private volatile long timeOfOldestEdit; - public final static long FIXED_OVERHEAD = ClassSize.align( - ClassSize.OBJECT + - (4 * ClassSize.REFERENCE) + - (2 * Bytes.SIZEOF_LONG)); - - public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + - (ClassSize.ATOMIC_LONG + ClassSize.TIMERANGE_TRACKER + - ClassSize.CELL_SET + ClassSize.CONCURRENT_SKIPLISTMAP)); + public final static long FIXED_OVERHEAD = ClassSize + .align(ClassSize.OBJECT + (4 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG)); + public final static long DEEP_OVERHEAD = FIXED_OVERHEAD; protected AbstractMemStore(final Configuration conf, final CellComparator c) { this.conf = conf; this.comparator = c; resetActive(); - this.snapshot = SegmentFactory.instance().createImmutableSegment(c, 0); + this.snapshot = SegmentFactory.instance().createImmutableSegment(c); this.snapshotId = NO_SNAPSHOT_ID; } protected void resetActive() { // Reset heap to not include any keys - this.active = SegmentFactory.instance().createMutableSegment(conf, comparator, DEEP_OVERHEAD); + this.active = SegmentFactory.instance().createMutableSegment(conf, comparator); this.timeOfOldestEdit = Long.MAX_VALUE; } @@ -200,8 +195,7 @@ public abstract class AbstractMemStore implements MemStore { // create a new snapshot and let the old one go. Segment oldSnapshot = this.snapshot; if (!this.snapshot.isEmpty()) { - this.snapshot = SegmentFactory.instance().createImmutableSegment( - getComparator(), 0); + this.snapshot = SegmentFactory.instance().createImmutableSegment(this.comparator); } this.snapshotId = NO_SNAPSHOT_ID; oldSnapshot.close(); @@ -213,12 +207,12 @@ public abstract class AbstractMemStore implements MemStore { */ @Override public long heapSize() { - return getActive().getSize(); + return size(); } @Override public long getSnapshotSize() { - return getSnapshot().getSize(); + return this.snapshot.keySize(); } @Override @@ -385,7 +379,7 @@ public abstract class AbstractMemStore implements MemStore { // so we cant add the new Cell w/o knowing what's there already, but we also // want to take this chance to delete some cells. So two loops (sad) - SortedSet<Cell> ss = getActive().tailSet(firstCell); + SortedSet<Cell> ss = this.active.tailSet(firstCell); for (Cell cell : ss) { // if this isnt the row we are interested in, then bail: if (!CellUtil.matchingColumn(cell, family, qualifier) @@ -433,38 +427,33 @@ public abstract class AbstractMemStore implements MemStore { } } + /** + * @return The size of the active segment. Means sum of all cell's size. + */ protected long keySize() { - return heapSize() - DEEP_OVERHEAD; + return this.active.keySize(); } protected CellComparator getComparator() { return comparator; } - protected MutableSegment getActive() { + @VisibleForTesting + MutableSegment getActive() { return active; } - protected ImmutableSegment getSnapshot() { + @VisibleForTesting + ImmutableSegment getSnapshot() { return snapshot; } - protected AbstractMemStore setSnapshot(ImmutableSegment snapshot) { - this.snapshot = snapshot; - return this; - } - - protected void setSnapshotSize(long snapshotSize) { - getSnapshot().setSize(snapshotSize); - } - /** * Check whether anything need to be done based on the current active set size */ protected abstract void checkActiveSize(); /** - * Returns an ordered list of segments from most recent to oldest in memstore * @return an ordered list of segments from most recent to oldest in memstore */ protected abstract List<Segment> getSegments() throws IOException; http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 504ddab..177f222 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; @@ -52,13 +51,6 @@ import org.apache.hadoop.hbase.wal.WAL; @InterfaceAudience.Private public class CompactingMemStore extends AbstractMemStore { - public final static long DEEP_OVERHEAD_PER_PIPELINE_SKIPLIST_ITEM = ClassSize.align( - ClassSize.TIMERANGE_TRACKER + ClassSize.CELL_SET + ClassSize.CONCURRENT_SKIPLISTMAP); - - public final static long DEEP_OVERHEAD_PER_PIPELINE_CELL_ARRAY_ITEM = ClassSize.align( - ClassSize.TIMERANGE_TRACKER + ClassSize.TIMERANGE + - ClassSize.CELL_SET + ClassSize.CELL_ARRAY_MAP); - // Default fraction of in-memory-flush size w.r.t. flush-to-disk size public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY = "hbase.memstore.inmemoryflush.threshold.factor"; @@ -75,6 +67,13 @@ public class CompactingMemStore extends AbstractMemStore { @VisibleForTesting private final AtomicBoolean allowCompaction = new AtomicBoolean(true); + public static final long DEEP_OVERHEAD = AbstractMemStore.DEEP_OVERHEAD + + 6 * ClassSize.REFERENCE // Store, RegionServicesForStores, CompactionPipeline, + // MemStoreCompactor, inMemoryFlushInProgress, allowCompaction + + Bytes.SIZEOF_LONG // inmemoryFlushSize + + 2 * ClassSize.ATOMIC_BOOLEAN// inMemoryFlushInProgress and allowCompaction + + CompactionPipeline.DEEP_OVERHEAD + MemStoreCompactor.DEEP_OVERHEAD; + public CompactingMemStore(Configuration conf, CellComparator c, HStore store, RegionServicesForStores regionServices) throws IOException { super(conf, c); @@ -100,28 +99,18 @@ public class CompactingMemStore extends AbstractMemStore { LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize); } - public static long getSegmentSize(Segment segment) { - return segment.keySize(); - } - - public static long getSegmentsSize(List<? extends Segment> list) { - long res = 0; - for (Segment segment : list) { - res += getSegmentSize(segment); - } - return res; - } - /** - * @return Total memory occupied by this MemStore. - * This is not thread safe and the memstore may be changed while computing its size. - * It is the responsibility of the caller to make sure this doesn't happen. + * @return Total memory occupied by this MemStore. This includes active segment size and heap size + * overhead of this memstore but won't include any size occupied by the snapshot. We + * assume the snapshot will get cleared soon. This is not thread safe and the memstore may + * be changed while computing its size. It is the responsibility of the caller to make + * sure this doesn't happen. */ @Override public long size() { - long res = 0; - for (Segment item : getSegments()) { - res += item.getSize(); + long res = DEEP_OVERHEAD + this.active.size(); + for (Segment item : pipeline.getSegments()) { + res += CompactionPipeline.ENTRY_OVERHEAD + item.size(); } return res; } @@ -131,11 +120,13 @@ public class CompactingMemStore extends AbstractMemStore { * The store may do any post-flush actions at this point. * One example is to update the WAL with sequence number that is known only at the store level. */ - @Override public void finalizeFlush() { + @Override + public void finalizeFlush() { updateLowestUnflushedSequenceIdInWAL(false); } - @Override public boolean isSloppy() { + @Override + public boolean isSloppy() { return true; } @@ -148,10 +139,9 @@ public class CompactingMemStore extends AbstractMemStore { */ @Override public MemStoreSnapshot snapshot() { - MutableSegment active = getActive(); // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. - if (!getSnapshot().isEmpty()) { + if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { @@ -161,21 +151,22 @@ public class CompactingMemStore extends AbstractMemStore { + getFamilyName()); } stopCompaction(); - pushActiveToPipeline(active); + pushActiveToPipeline(this.active); snapshotId = EnvironmentEdgeManager.currentTime(); pushTailToSnapshot(); } - return new MemStoreSnapshot(snapshotId, getSnapshot()); + return new MemStoreSnapshot(snapshotId, this.snapshot); } /** * On flush, how much memory we will clear. * @return size of data that is going to be flushed */ - @Override public long getFlushableSize() { - long snapshotSize = getSnapshot().getSize(); - if(snapshotSize == 0) { - //if snapshot is empty the tail of the pipeline is flushed + @Override + public long getFlushableSize() { + long snapshotSize = getSnapshotSize(); + if (snapshotSize == 0) { + // if snapshot is empty the tail of the pipeline is flushed snapshotSize = pipeline.getTailSize(); } return snapshotSize > 0 ? snapshotSize : keySize(); @@ -186,7 +177,7 @@ public class CompactingMemStore extends AbstractMemStore { long minSequenceId = pipeline.getMinSequenceId(); if(minSequenceId != Long.MAX_VALUE) { byte[] encodedRegionName = getRegionServices().getRegionInfo().getEncodedNameAsBytes(); - byte[] familyName = getFamilyNameInByte(); + byte[] familyName = getFamilyNameInBytes(); WAL WAL = getRegionServices().getWAL(); if (WAL != null) { WAL.updateStore(encodedRegionName, familyName, minSequenceId, onlyIfGreater); @@ -197,10 +188,10 @@ public class CompactingMemStore extends AbstractMemStore { @Override public List<Segment> getSegments() { List<Segment> pipelineList = pipeline.getSegments(); - List<Segment> list = new LinkedList<Segment>(); - list.add(getActive()); + List<Segment> list = new ArrayList<Segment>(pipelineList.size() + 2); + list.add(this.active); list.addAll(pipelineList); - list.add(getSnapshot()); + list.add(this.snapshot); return list; } @@ -235,7 +226,7 @@ public class CompactingMemStore extends AbstractMemStore { } public String getFamilyName() { - return Bytes.toString(getFamilyNameInByte()); + return Bytes.toString(getFamilyNameInBytes()); } @Override @@ -248,12 +239,12 @@ public class CompactingMemStore extends AbstractMemStore { // The list of elements in pipeline + the active element + the snapshot segment // TODO : This will change when the snapshot is made of more than one element List<KeyValueScanner> list = new ArrayList<KeyValueScanner>(pipelineList.size() + 2); - list.add(getActive().getScanner(readPt, order + 1)); + list.add(this.active.getScanner(readPt, order + 1)); for (Segment item : pipelineList) { list.add(item.getScanner(readPt, order)); order--; } - list.add(getSnapshot().getScanner(readPt, order)); + list.add(this.snapshot.getScanner(readPt, order)); return Collections.<KeyValueScanner> singletonList(new MemStoreScanner(getComparator(), list)); } @@ -291,11 +282,10 @@ public class CompactingMemStore extends AbstractMemStore { // Phase I: Update the pipeline getRegionServices().blockUpdates(); try { - MutableSegment active = getActive(); if (LOG.isDebugEnabled()) { LOG.debug("IN-MEMORY FLUSH: Pushing active segment into compaction pipeline"); } - pushActiveToPipeline(active); + pushActiveToPipeline(this.active); } finally { getRegionServices().unblockUpdates(); } @@ -319,7 +309,7 @@ public class CompactingMemStore extends AbstractMemStore { } } - private byte[] getFamilyNameInByte() { + private byte[] getFamilyNameInBytes() { return store.getFamily().getName(); } @@ -328,7 +318,7 @@ public class CompactingMemStore extends AbstractMemStore { } private boolean shouldFlushInMemory() { - if(getActive().getSize() > inmemoryFlushSize) { // size above flush threshold + if (this.active.size() > inmemoryFlushSize) { // size above flush threshold // the inMemoryFlushInProgress is CASed to be true here in order to mutual exclude // the insert of the active into the compaction pipeline return (inMemoryFlushInProgress.compareAndSet(false,true)); @@ -350,8 +340,6 @@ public class CompactingMemStore extends AbstractMemStore { private void pushActiveToPipeline(MutableSegment active) { if (!active.isEmpty()) { - long delta = DEEP_OVERHEAD_PER_PIPELINE_SKIPLIST_ITEM - DEEP_OVERHEAD; - active.incSize(delta); pipeline.pushHead(active); resetActive(); } @@ -360,9 +348,7 @@ public class CompactingMemStore extends AbstractMemStore { private void pushTailToSnapshot() { ImmutableSegment tail = pipeline.pullTail(); if (!tail.isEmpty()) { - setSnapshot(tail); - long size = getSegmentSize(tail); - setSnapshotSize(size); + this.snapshot = tail; } } @@ -428,7 +414,7 @@ public class CompactingMemStore extends AbstractMemStore { // debug method public void debug() { - String msg = "active size="+getActive().getSize(); + String msg = "active size=" + this.active.size(); msg += " threshold="+IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT* inmemoryFlushSize; msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false"); msg += " inMemoryFlushInProgress is "+ (inMemoryFlushInProgress.get() ? "true" : "false"); http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java index e0ba8c3..6a13f43 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java @@ -24,7 +24,10 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; /** * The compaction pipeline of a {@link CompactingMemStore}, is a FIFO queue of segments. @@ -39,13 +42,17 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; public class CompactionPipeline { private static final Log LOG = LogFactory.getLog(CompactionPipeline.class); + public final static long FIXED_OVERHEAD = ClassSize + .align(ClassSize.OBJECT + (2 * ClassSize.REFERENCE) + Bytes.SIZEOF_LONG); + public final static long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.LINKEDLIST; + public final static long ENTRY_OVERHEAD = ClassSize.LINKEDLIST_ENTRY; + private final RegionServicesForStores region; private LinkedList<ImmutableSegment> pipeline; private long version; private static final ImmutableSegment EMPTY_MEM_STORE_SEGMENT = SegmentFactory.instance() - .createImmutableSegment(null, - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_SKIPLIST_ITEM); + .createImmutableSegment((CellComparator) null); public CompactionPipeline(RegionServicesForStores region) { this.region = region; @@ -105,8 +112,8 @@ public class CompactionPipeline { } if (region != null) { // update the global memstore size counter - long suffixSize = CompactingMemStore.getSegmentsSize(suffix); - long newSize = CompactingMemStore.getSegmentSize(segment); + long suffixSize = getSegmentsKeySize(suffix); + long newSize = segment.keySize(); long delta = suffixSize - newSize; long globalMemstoreSize = region.addAndGetGlobalMemstoreSize(-delta); if (LOG.isDebugEnabled()) { @@ -117,6 +124,14 @@ public class CompactionPipeline { return true; } + private static long getSegmentsKeySize(List<? extends Segment> list) { + long res = 0; + for (Segment segment : list) { + res += segment.keySize(); + } + return res; + } + /** * If the caller holds the current version, go over the the pipeline and try to flatten each * segment. Flattening is replacing the ConcurrentSkipListMap based CellSet to CellArrayMap based. @@ -178,20 +193,20 @@ public class CompactionPipeline { public long getMinSequenceId() { long minSequenceId = Long.MAX_VALUE; - if(!isEmpty()) { + if (!isEmpty()) { minSequenceId = pipeline.getLast().getMinSequenceId(); } return minSequenceId; } public long getTailSize() { - if(isEmpty()) return 0; - return CompactingMemStore.getSegmentSize(pipeline.peekLast()); + if (isEmpty()) return 0; + return pipeline.peekLast().keySize(); } private void swapSuffix(LinkedList<ImmutableSegment> suffix, ImmutableSegment segment) { version++; - for(Segment itemInSuffix : suffix) { + for (Segment itemInSuffix : suffix) { itemInSuffix.close(); } pipeline.removeAll(suffix); http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 50ab06b..b448b04 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -83,20 +83,19 @@ public class DefaultMemStore extends AbstractMemStore { public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. - if (!getSnapshot().isEmpty()) { + if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); - if (!getActive().isEmpty()) { + if (!this.active.isEmpty()) { ImmutableSegment immutableSegment = SegmentFactory.instance(). - createImmutableSegment(getActive()); - setSnapshot(immutableSegment); - setSnapshotSize(keySize()); + createImmutableSegment(this.active); + this.snapshot = immutableSegment; resetActive(); } } - return new MemStoreSnapshot(this.snapshotId, getSnapshot()); + return new MemStoreSnapshot(this.snapshotId, this.snapshot); } /** @@ -106,7 +105,7 @@ public class DefaultMemStore extends AbstractMemStore { */ @Override public long getFlushableSize() { - long snapshotSize = getSnapshot().getSize(); + long snapshotSize = getSnapshotSize(); return snapshotSize > 0 ? snapshotSize : keySize(); } @@ -116,8 +115,8 @@ public class DefaultMemStore extends AbstractMemStore { */ public List<KeyValueScanner> getScanners(long readPt) throws IOException { List<KeyValueScanner> list = new ArrayList<KeyValueScanner>(2); - list.add(getActive().getScanner(readPt, 1)); - list.add(getSnapshot().getScanner(readPt, 0)); + list.add(this.active.getScanner(readPt, 1)); + list.add(this.snapshot.getScanner(readPt, 0)); return Collections.<KeyValueScanner> singletonList( new MemStoreScanner(getComparator(), list)); } @@ -125,8 +124,8 @@ public class DefaultMemStore extends AbstractMemStore { @Override protected List<Segment> getSegments() throws IOException { List<Segment> list = new ArrayList<Segment>(2); - list.add(getActive()); - list.add(getSnapshot()); + list.add(this.active); + list.add(this.snapshot); return list; } @@ -137,19 +136,16 @@ public class DefaultMemStore extends AbstractMemStore { */ Cell getNextRow(final Cell cell) { return getLowest( - getNextRow(cell, getActive().getCellSet()), - getNextRow(cell, getSnapshot().getCellSet())); + getNextRow(cell, this.active.getCellSet()), + getNextRow(cell, this.snapshot.getCellSet())); } @Override public void updateLowestUnflushedSequenceIdInWAL(boolean onlyIfMoreRecent) { } - /** - * @return Total memory occupied by this MemStore. - */ @Override public long size() { - return heapSize(); + return this.active.size() + DEEP_OVERHEAD; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index 28f14d5..12b7916 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.client.Scan; @@ -37,11 +38,12 @@ import java.io.IOException; */ @InterfaceAudience.Private public class ImmutableSegment extends Segment { - /** - * This is an immutable segment so use the read-only TimeRange rather than the heavy-weight - * TimeRangeTracker with all its synchronization when doing time range stuff. - */ - private final TimeRange timeRange; + + private static final long DEEP_OVERHEAD = Segment.DEEP_OVERHEAD + + (2 * ClassSize.REFERENCE) // Refs to timeRange and type + + ClassSize.TIMERANGE; + public static final long DEEP_OVERHEAD_CSLM = DEEP_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP; + public static final long DEEP_OVERHEAD_CAM = DEEP_OVERHEAD + ClassSize.CELL_ARRAY_MAP; /** * Types of ImmutableSegment @@ -51,6 +53,12 @@ public class ImmutableSegment extends Segment { ARRAY_MAP_BASED, } + /** + * This is an immutable segment so use the read-only TimeRange rather than the heavy-weight + * TimeRangeTracker with all its synchronization when doing time range stuff. + */ + private final TimeRange timeRange; + private Type type = Type.SKIPLIST_MAP_BASED; // whether it is based on CellFlatMap or ConcurrentSkipListMap @@ -66,9 +74,8 @@ public class ImmutableSegment extends Segment { */ protected ImmutableSegment(Segment segment) { super(segment); - type = Type.SKIPLIST_MAP_BASED; - TimeRangeTracker trt = getTimeRangeTracker(); - this.timeRange = trt == null? null: trt.toTimeRange(); + this.type = Type.SKIPLIST_MAP_BASED; + this.timeRange = this.timeRangeTracker == null ? null : this.timeRangeTracker.toTimeRange(); } /**------------------------------------------------------------------------ @@ -80,20 +87,14 @@ public class ImmutableSegment extends Segment { */ protected ImmutableSegment(CellComparator comparator, MemStoreCompactorIterator iterator, MemStoreLAB memStoreLAB, int numOfCells, Type type) { - - super(null, // initiailize the CellSet with NULL - comparator, memStoreLAB, - // initial size of segment metadata (the data per cell is added in createCellArrayMapSet) - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_CELL_ARRAY_ITEM, - ClassSize.CELL_ARRAY_MAP_ENTRY); - + super(null, // initiailize the CellSet with NULL + comparator, memStoreLAB); + this.type = type; // build the true CellSet based on CellArrayMap CellSet cs = createCellArrayMapSet(numOfCells, iterator); this.setCellSet(null, cs); // update the CellSet of the new Segment - this.type = type; - TimeRangeTracker trt = getTimeRangeTracker(); - this.timeRange = trt == null? null: trt.toTimeRange(); + this.timeRange = this.timeRangeTracker == null ? null : this.timeRangeTracker.toTimeRange(); } /**------------------------------------------------------------------------ @@ -101,15 +102,11 @@ public class ImmutableSegment extends Segment { * list of older ImmutableSegments. * The given iterator returns the Cells that "survived" the compaction. */ - protected ImmutableSegment( - CellComparator comparator, MemStoreCompactorIterator iterator, MemStoreLAB memStoreLAB) { - - super(new CellSet(comparator), // initiailize the CellSet with empty CellSet - comparator, memStoreLAB, - // initial size of segment metadata (the data per cell is added in internalAdd) - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_SKIPLIST_ITEM, - ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); - + protected ImmutableSegment(CellComparator comparator, MemStoreCompactorIterator iterator, + MemStoreLAB memStoreLAB) { + super(new CellSet(comparator), // initiailize the CellSet with empty CellSet + comparator, memStoreLAB); + type = Type.SKIPLIST_MAP_BASED; while (iterator.hasNext()) { Cell c = iterator.next(); // The scanner is doing all the elimination logic @@ -118,9 +115,7 @@ public class ImmutableSegment extends Segment { boolean usedMSLAB = (newKV != c); internalAdd(newKV, usedMSLAB); // } - type = Type.SKIPLIST_MAP_BASED; - TimeRangeTracker trt = getTimeRangeTracker(); - this.timeRange = trt == null? null: trt.toTimeRange(); + this.timeRange = this.timeRangeTracker == null ? null : this.timeRangeTracker.toTimeRange(); } ///////////////////// PUBLIC METHODS ///////////////////// @@ -144,14 +139,16 @@ public class ImmutableSegment extends Segment { return this.timeRange.getMin(); } + @Override - public long keySize() { - switch (type){ + public long size() { + switch (this.type) { case SKIPLIST_MAP_BASED: - return size.get() - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_SKIPLIST_ITEM; + return keySize() + DEEP_OVERHEAD_CSLM; case ARRAY_MAP_BASED: - return size.get() - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_CELL_ARRAY_ITEM; - default: throw new IllegalStateException(); + return keySize() + DEEP_OVERHEAD_CAM; + default: + throw new RuntimeException("Unknown type " + type); } } @@ -171,9 +168,6 @@ public class ImmutableSegment extends Segment { CellSet oldCellSet = getCellSet(); int numOfCells = getCellsCount(); - // each Cell is now represented in CellArrayMap - constantCellMetaDataSize = ClassSize.CELL_ARRAY_MAP_ENTRY; - // build the new (CellSet CellArrayMap based) CellSet newCellSet = recreateCellArrayMapSet(numOfCells); type = Type.ARRAY_MAP_BASED; @@ -214,6 +208,19 @@ public class ImmutableSegment extends Segment { return new CellSet(cam); } + protected long heapSizeChange(Cell cell, boolean succ) { + if (succ) { + switch (this.type) { + case SKIPLIST_MAP_BASED: + return ClassSize + .align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + CellUtil.estimatedHeapSizeOf(cell)); + case ARRAY_MAP_BASED: + return ClassSize.align(ClassSize.CELL_ARRAY_MAP_ENTRY + CellUtil.estimatedHeapSizeOf(cell)); + } + } + return 0; + } + /*------------------------------------------------------------------------*/ // Create CellSet based on CellArrayMap from current ConcurrentSkipListMap based CellSet // (without compacting iterator) @@ -239,5 +246,4 @@ public class ImmutableSegment extends Segment { CellArrayMap cam = new CellArrayMap(getComparator(), cells, 0, idx, false); return new CellSet(cam); } - } http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java index 52ae6a3..d52b863 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java @@ -126,7 +126,11 @@ public interface MemStore extends HeapSize { List<KeyValueScanner> getScanners(long readPt) throws IOException; /** - * @return Total memory occupied by this MemStore. + * @return Total memory occupied by this MemStore. This includes active segment size and heap size + * overhead of this memstore but won't include any size occupied by the snapshot. We + * assume the snapshot will get cleared soon. This is not thread safe and the memstore may + * be changed while computing its size. It is the responsibility of the caller to make + * sure this doesn't happen. */ long size(); http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java index 470dc9c..714ffe3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java @@ -22,6 +22,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; @@ -38,7 +40,11 @@ import java.util.concurrent.atomic.AtomicBoolean; * therefore no special synchronization is required. */ @InterfaceAudience.Private -class MemStoreCompactor { +public class MemStoreCompactor { + + public static final long DEEP_OVERHEAD = ClassSize + .align(ClassSize.OBJECT + 4 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT + Bytes.SIZEOF_DOUBLE + + ClassSize.ATOMIC_BOOLEAN); // Option for external guidance whether flattening is allowed static final String MEMSTORE_COMPACTOR_FLATTENING = "hbase.hregion.compacting.memstore.flatten"; @@ -59,6 +65,15 @@ class MemStoreCompactor { static final boolean MEMSTORE_COMPACTOR_AVOID_SPECULATIVE_SCAN_DEFAULT = false; private static final Log LOG = LogFactory.getLog(MemStoreCompactor.class); + + /** + * Types of Compaction + */ + private enum Type { + COMPACT_TO_SKIPLIST_MAP, + COMPACT_TO_ARRAY_MAP + } + private CompactingMemStore compactingMemStore; // a static version of the segment list from the pipeline @@ -73,13 +88,6 @@ class MemStoreCompactor { double fraction = 0.8; int immutCellsNum = 0; // number of immutable for compaction cells - /** - * Types of Compaction - */ - private enum Type { - COMPACT_TO_SKIPLIST_MAP, - COMPACT_TO_ARRAY_MAP - } private Type type = Type.COMPACT_TO_ARRAY_MAP; http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java index f64979f..1bb4511 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java @@ -36,7 +36,7 @@ public class MemStoreSnapshot { public MemStoreSnapshot(long id, ImmutableSegment snapshot) { this.id = id; this.cellsCount = snapshot.getCellsCount(); - this.size = snapshot.getSize(); + this.size = snapshot.keySize(); this.timeRangeTracker = snapshot.getTimeRangeTracker(); this.scanner = snapshot.getKeyValueScanner(); this.tagsPresent = snapshot.isTagsPresent(); http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java index 1cac7fd..3fb9723 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java @@ -21,17 +21,21 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.ClassSize; + +import com.google.common.annotations.VisibleForTesting; /** * A mutable segment in memstore, specifically the active segment. */ @InterfaceAudience.Private public class MutableSegment extends Segment { - protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, - long size) { - super(cellSet, comparator, memStoreLAB, size, ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); + + public final static long DEEP_OVERHEAD = Segment.DEEP_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP; + + protected MutableSegment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB) { + super(cellSet, comparator, memStoreLAB); } /** @@ -44,29 +48,28 @@ public class MutableSegment extends Segment { return internalAdd(cell, mslabUsed); } - //methods for test - /** * Returns the first cell in the segment * @return the first cell in the segment */ + @VisibleForTesting Cell first() { return this.getCellSet().first(); } @Override public boolean shouldSeek(Scan scan, long oldestUnexpiredTS) { - return (getTimeRangeTracker().includesTimeRange(scan.getTimeRange()) - && (getTimeRangeTracker().getMax() >= oldestUnexpiredTS)); + return (this.timeRangeTracker.includesTimeRange(scan.getTimeRange()) + && (this.timeRangeTracker.getMax() >= oldestUnexpiredTS)); } @Override public long getMinTimestamp() { - return getTimeRangeTracker().getMin(); + return this.timeRangeTracker.getMin(); } @Override - public long keySize() { - return size.get() - CompactingMemStore.DEEP_OVERHEAD_PER_PIPELINE_SKIPLIST_ITEM; + public long size() { + return keySize() + DEEP_OVERHEAD; } } http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index b4d1d61..01f3da9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -24,7 +24,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; @@ -33,6 +32,7 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.ByteRange; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import com.google.common.annotations.VisibleForTesting; @@ -47,28 +47,31 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceAudience.Private public abstract class Segment { - private static final Log LOG = LogFactory.getLog(Segment.class); + final static long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + + 5 * ClassSize.REFERENCE // cellSet, comparator, memStoreLAB, size, timeRangeTracker + + Bytes.SIZEOF_LONG // minSequenceId + + Bytes.SIZEOF_BOOLEAN); // tagsPresent + public final static long DEEP_OVERHEAD = FIXED_OVERHEAD + ClassSize.ATOMIC_REFERENCE + + ClassSize.CELL_SET + ClassSize.ATOMIC_LONG + ClassSize.TIMERANGE_TRACKER; + private AtomicReference<CellSet> cellSet= new AtomicReference<CellSet>(); private final CellComparator comparator; private long minSequenceId; - private volatile MemStoreLAB memStoreLAB; - /* The size includes everything allocated for this segment, - * use keySize() to get only size of the cells */ + private MemStoreLAB memStoreLAB; + // Sum of sizes of all Cells added to this Segment. Cell's heapSize is considered. This is not + // including the heap overhead of this class. protected final AtomicLong size; + protected final TimeRangeTracker timeRangeTracker; protected volatile boolean tagsPresent; - private final TimeRangeTracker timeRangeTracker; - protected long constantCellMetaDataSize; - protected Segment( - CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, long size, - long constantCellSize) { + // This constructor is used to create empty Segments. + protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB) { this.cellSet.set(cellSet); this.comparator = comparator; this.minSequenceId = Long.MAX_VALUE; this.memStoreLAB = memStoreLAB; - this.size = new AtomicLong(size); + this.size = new AtomicLong(0); this.tagsPresent = false; - this.constantCellMetaDataSize = constantCellSize; this.timeRangeTracker = new TimeRangeTracker(); } @@ -77,9 +80,8 @@ public abstract class Segment { this.comparator = segment.getComparator(); this.minSequenceId = segment.getMinSequenceId(); this.memStoreLAB = segment.getMemStoreLAB(); - this.size = new AtomicLong(segment.getSize()); + this.size = new AtomicLong(segment.keySize()); this.tagsPresent = segment.isTagsPresent(); - this.constantCellMetaDataSize = segment.getConstantCellMetaDataSize(); this.timeRangeTracker = segment.getTimeRangeTracker(); } @@ -100,7 +102,6 @@ public abstract class Segment { } /** - * Returns whether the segment has any cells * @return whether the segment has any cells */ public boolean isEmpty() { @@ -108,7 +109,6 @@ public abstract class Segment { } /** - * Returns number of cells in segment * @return number of cells in segment */ public int getCellsCount() { @@ -116,7 +116,6 @@ public abstract class Segment { } /** - * Returns the first cell in the segment that has equal or greater key than the given cell * @return the first cell in the segment that has equal or greater key than the given cell */ public Cell getFirstAfter(Cell cell) { @@ -131,9 +130,8 @@ public abstract class Segment { * Closing a segment before it is being discarded */ public void close() { - MemStoreLAB mslab = getMemStoreLAB(); - if(mslab != null) { - mslab.close(); + if (this.memStoreLAB != null) { + this.memStoreLAB.close(); } // do not set MSLab to null as scanners may still be reading the data here and need to decrease // the counter when they finish @@ -145,12 +143,12 @@ public abstract class Segment { * @return either the given cell or its clone */ public Cell maybeCloneWithAllocator(Cell cell) { - if (getMemStoreLAB() == null) { + if (this.memStoreLAB == null) { return cell; } int len = getCellLength(cell); - ByteRange alloc = getMemStoreLAB().allocateBytes(len); + ByteRange alloc = this.memStoreLAB.allocateBytes(len); if (alloc == null) { // The allocation was too large, allocator decided // not to do anything with it. @@ -180,28 +178,18 @@ public abstract class Segment { } public void incScannerCount() { - if(getMemStoreLAB() != null) { - getMemStoreLAB().incScannerCount(); + if (this.memStoreLAB != null) { + this.memStoreLAB.incScannerCount(); } } public void decScannerCount() { - if(getMemStoreLAB() != null) { - getMemStoreLAB().decScannerCount(); + if (this.memStoreLAB != null) { + this.memStoreLAB.decScannerCount(); } } /** - * Setting the heap size of the segment - used to account for different class overheads - * @return this object - */ - - public Segment setSize(long size) { - this.size.set(size); - return this; - } - - /** * Setting the CellSet of the segment - used only for flat immutable segment for setting * immutable CellSet after its creation in immutable segment constructor * @return this object @@ -212,22 +200,23 @@ public abstract class Segment { return this; } - /* return only cell's heap size */ - public abstract long keySize(); + /** + * @return Sum of all cell's size. + */ + public long keySize() { + return this.size.get(); + } /** - * Returns the heap size of the segment * @return the heap size of the segment */ - public long getSize() { - return size.get(); - } + public abstract long size(); /** * Updates the heap size counter of the segment by the given delta */ public void incSize(long delta) { - size.addAndGet(delta); + this.size.addAndGet(delta); } public long getMinSequenceId() { @@ -260,7 +249,6 @@ public abstract class Segment { } /** - * Returns a set of all cells in the segment * @return a set of all cells in the segment */ protected CellSet getCellSet() { @@ -302,6 +290,11 @@ public abstract class Segment { return s; } + protected long heapSizeChange(Cell cell, boolean succ) { + return succ ? ClassSize + .align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + CellUtil.estimatedHeapSizeOf(cell)) : 0; + } + /** * Returns a subset of the segment cell set, which starts with the given cell * @param firstCell a cell in the segment @@ -312,7 +305,7 @@ public abstract class Segment { } @VisibleForTesting - public MemStoreLAB getMemStoreLAB() { + MemStoreLAB getMemStoreLAB() { return memStoreLAB; } @@ -326,29 +319,13 @@ public abstract class Segment { } } - /* - * Calculate how the MemStore size has changed. Includes overhead of the - * backing Map. - * @param cell - * @param notPresent True if the cell was NOT present in the set. - * @return change in size - */ - protected long heapSizeChange(final Cell cell, final boolean notPresent){ - return - notPresent ? - ClassSize.align(constantCellMetaDataSize + CellUtil.estimatedHeapSizeOf(cell)) : 0; - } - - public long getConstantCellMetaDataSize() { - return this.constantCellMetaDataSize; - } - @Override public String toString() { String res = "Store segment of type "+this.getClass().getName()+"; "; res += "isEmpty "+(isEmpty()?"yes":"no")+"; "; res += "cellCount "+getCellsCount()+"; "; - res += "size "+getSize()+"; "; + res += "cellsSize "+keySize()+"; "; + res += "heapSize "+size()+"; "; res += "Min ts "+getMinTimestamp()+"; "; return res; } http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java index 6351f13..510ebbd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentFactory.java @@ -39,6 +39,7 @@ public final class SegmentFactory { private SegmentFactory() {} private static SegmentFactory instance = new SegmentFactory(); + public static SegmentFactory instance() { return instance; } @@ -46,47 +47,43 @@ public final class SegmentFactory { // create skip-list-based (non-flat) immutable segment from compacting old immutable segments public ImmutableSegment createImmutableSegment(final Configuration conf, final CellComparator comparator, MemStoreCompactorIterator iterator) { - MemStoreLAB memStoreLAB = getMemStoreLAB(conf); - return - new ImmutableSegment(comparator, iterator, memStoreLAB); + return new ImmutableSegment(comparator, iterator, getMemStoreLAB(conf)); + } + + // create new flat immutable segment from compacting old immutable segment + public ImmutableSegment createImmutableSegment(final Configuration conf, + final CellComparator comparator, MemStoreCompactorIterator iterator, int numOfCells, + ImmutableSegment.Type segmentType) throws IOException { + Preconditions.checkArgument(segmentType != ImmutableSegment.Type.SKIPLIST_MAP_BASED, + "wrong immutable segment type"); + return new ImmutableSegment(comparator, iterator, getMemStoreLAB(conf), numOfCells, + segmentType); } // create empty immutable segment - public ImmutableSegment createImmutableSegment(CellComparator comparator, long size) { - MutableSegment segment = generateMutableSegment(null, comparator, null, size); + public ImmutableSegment createImmutableSegment(CellComparator comparator) { + MutableSegment segment = generateMutableSegment(null, comparator, null); return createImmutableSegment(segment); } - // create immutable segment from mutable + // create immutable segment from mutable segment public ImmutableSegment createImmutableSegment(MutableSegment segment) { return new ImmutableSegment(segment); } // create mutable segment - public MutableSegment createMutableSegment(final Configuration conf, - CellComparator comparator, long size) { - MemStoreLAB memStoreLAB = getMemStoreLAB(conf); - return generateMutableSegment(conf, comparator, memStoreLAB, size); - } - - // create new flat immutable segment from compacting old immutable segment - public ImmutableSegment createImmutableSegment(final Configuration conf, final CellComparator comparator, - MemStoreCompactorIterator iterator, int numOfCells, ImmutableSegment.Type segmentType) - throws IOException { - Preconditions.checkArgument( - segmentType != ImmutableSegment.Type.SKIPLIST_MAP_BASED, "wrong immutable segment type"); + public MutableSegment createMutableSegment(final Configuration conf, CellComparator comparator) { MemStoreLAB memStoreLAB = getMemStoreLAB(conf); - return - new ImmutableSegment(comparator, iterator, memStoreLAB, numOfCells, segmentType); + return generateMutableSegment(conf, comparator, memStoreLAB); } //****** private methods to instantiate concrete store segments **********// - private MutableSegment generateMutableSegment( - final Configuration conf, CellComparator comparator, MemStoreLAB memStoreLAB, long size) { + private MutableSegment generateMutableSegment(final Configuration conf, CellComparator comparator, + MemStoreLAB memStoreLAB) { // TBD use configuration to set type of segment CellSet set = new CellSet(comparator); - return new MutableSegment(set, comparator, memStoreLAB, size); + return new MutableSegment(set, comparator, memStoreLAB); } private MemStoreLAB getMemStoreLAB(Configuration conf) { http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index e7d6661..6e8f831 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -40,6 +40,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.LinkedList; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -49,6 +50,7 @@ import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.junit.Assert.assertEquals; @@ -310,16 +312,104 @@ public class TestHeapSize { // DefaultMemStore Deep Overhead actual = DefaultMemStore.DEEP_OVERHEAD; expected = ClassSize.estimateBase(cl, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + assertEquals(expected, actual); + } + + // CompactingMemStore Deep Overhead + cl = CompactingMemStore.class; + actual = CompactingMemStore.DEEP_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + expected += ClassSize.estimateBase(AtomicBoolean.class, false); + expected += ClassSize.estimateBase(AtomicBoolean.class, false); + expected += ClassSize.estimateBase(CompactionPipeline.class, false); + expected += ClassSize.estimateBase(LinkedList.class, false); + expected += ClassSize.estimateBase(MemStoreCompactor.class, false); + expected += ClassSize.estimateBase(AtomicBoolean.class, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + ClassSize.estimateBase(AtomicBoolean.class, true); + ClassSize.estimateBase(AtomicBoolean.class, true); + ClassSize.estimateBase(CompactionPipeline.class, true); + ClassSize.estimateBase(LinkedList.class, true); + ClassSize.estimateBase(MemStoreCompactor.class, true); + ClassSize.estimateBase(AtomicBoolean.class, true); + assertEquals(expected, actual); + } + + // Segment Deep overhead + cl = Segment.class; + actual = Segment.DEEP_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); + expected += ClassSize.estimateBase(AtomicLong.class, false); + expected += ClassSize.estimateBase(AtomicReference.class, false); + expected += ClassSize.estimateBase(CellSet.class, false); + expected += ClassSize.estimateBase(TimeRangeTracker.class, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + ClassSize.estimateBase(AtomicLong.class, true); + ClassSize.estimateBase(AtomicReference.class, true); + ClassSize.estimateBase(CellSet.class, true); + ClassSize.estimateBase(TimeRangeTracker.class, true); + assertEquals(expected, actual); + } + + // MutableSegment Deep overhead + cl = MutableSegment.class; + actual = MutableSegment.DEEP_OVERHEAD; + expected = ClassSize.estimateBase(cl, false); expected += ClassSize.estimateBase(AtomicLong.class, false); + expected += ClassSize.estimateBase(AtomicReference.class, false); expected += ClassSize.estimateBase(CellSet.class, false); + expected += ClassSize.estimateBase(TimeRangeTracker.class, false); expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + ClassSize.estimateBase(AtomicLong.class, true); + ClassSize.estimateBase(AtomicReference.class, true); + ClassSize.estimateBase(CellSet.class, true); + ClassSize.estimateBase(TimeRangeTracker.class, true); + ClassSize.estimateBase(ConcurrentSkipListMap.class, true); + assertEquals(expected, actual); + } + + // ImmutableSegment Deep overhead + cl = ImmutableSegment.class; + actual = ImmutableSegment.DEEP_OVERHEAD_CSLM; + expected = ClassSize.estimateBase(cl, false); + expected += ClassSize.estimateBase(AtomicLong.class, false); + expected += ClassSize.estimateBase(AtomicReference.class, false); + expected += ClassSize.estimateBase(CellSet.class, false); expected += ClassSize.estimateBase(TimeRangeTracker.class, false); - if(expected != actual) { + expected += ClassSize.estimateBase(TimeRange.class, false); + expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false); + if (expected != actual) { ClassSize.estimateBase(cl, true); ClassSize.estimateBase(AtomicLong.class, true); + ClassSize.estimateBase(AtomicReference.class, true); ClassSize.estimateBase(CellSet.class, true); + ClassSize.estimateBase(TimeRangeTracker.class, true); + ClassSize.estimateBase(TimeRange.class, true); ClassSize.estimateBase(ConcurrentSkipListMap.class, true); + assertEquals(expected, actual); + } + actual = ImmutableSegment.DEEP_OVERHEAD_CAM; + expected = ClassSize.estimateBase(cl, false); + expected += ClassSize.estimateBase(AtomicLong.class, false); + expected += ClassSize.estimateBase(AtomicReference.class, false); + expected += ClassSize.estimateBase(CellSet.class, false); + expected += ClassSize.estimateBase(TimeRangeTracker.class, false); + expected += ClassSize.estimateBase(TimeRange.class, false); + expected += ClassSize.estimateBase(CellArrayMap.class, false); + if (expected != actual) { + ClassSize.estimateBase(cl, true); + ClassSize.estimateBase(AtomicLong.class, true); + ClassSize.estimateBase(AtomicReference.class, true); + ClassSize.estimateBase(CellSet.class, true); ClassSize.estimateBase(TimeRangeTracker.class, true); + ClassSize.estimateBase(TimeRange.class, true); + ClassSize.estimateBase(CellArrayMap.class, true); assertEquals(expected, actual); } http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index db0205e..211a6d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -377,7 +377,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { private long runSnapshot(final AbstractMemStore hmc, boolean useForce) throws IOException { // Save off old state. - long oldHistorySize = hmc.getSnapshot().getSize(); + long oldHistorySize = hmc.getSnapshot().keySize(); long prevTimeStamp = hmc.timeOfOldestEdit(); hmc.snapshot(); @@ -547,10 +547,6 @@ public class TestCompactingMemStore extends TestDefaultMemStore { memstore.add(new KeyValue(row, fam, qf1, 3, val)); assertEquals(3, memstore.getActive().getCellsCount()); - while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } - assertTrue(chunkPool.getPoolSize() == 0); // Chunks will be put back to pool after close scanners; @@ -597,9 +593,6 @@ public class TestCompactingMemStore extends TestDefaultMemStore { long size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact - while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(264, regionServicesForStores.getGlobalMemstoreTotalSize()); @@ -625,9 +618,6 @@ public class TestCompactingMemStore extends TestDefaultMemStore { long size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact - while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(1000); - } int counter = 0; for ( Segment s : memstore.getSegments()) { counter += s.getCellsCount(); @@ -641,9 +631,6 @@ public class TestCompactingMemStore extends TestDefaultMemStore { size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact - while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(368, regionServicesForStores.getGlobalMemstoreTotalSize()); @@ -672,9 +659,6 @@ public class TestCompactingMemStore extends TestDefaultMemStore { String tstStr = "\n\nFlushable size after first flush in memory:" + size + ". Is MemmStore in compaction?:" + ((CompactingMemStore)memstore).isMemStoreFlushingInMemory(); - while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) { - Threads.sleep(10); - } assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(264, regionServicesForStores.getGlobalMemstoreTotalSize()); @@ -719,7 +703,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { private void addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); - long size = hmc.getActive().getSize(); + long size = hmc.getActive().keySize(); for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts @@ -729,7 +713,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { hmc.add(kv); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } - regionServicesForStores.addAndGetGlobalMemstoreSize(hmc.getActive().getSize() - size); + regionServicesForStores.addAndGetGlobalMemstoreSize(hmc.getActive().keySize() - size); } private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java index 1933343..fefe2c1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java @@ -333,7 +333,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore private void addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); - long size = hmc.getActive().getSize();// + long size = hmc.getActive().size();// for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts @@ -343,7 +343,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore hmc.add(kv); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp()); } - regionServicesForStores.addAndGetGlobalMemstoreSize(hmc.getActive().getSize() - size);// + regionServicesForStores.addAndGetGlobalMemstoreSize(hmc.getActive().size() - size);// } private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { http://git-wip-us.apache.org/repos/asf/hbase/blob/77b32732/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java index 1615b99..6bfaa59 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java @@ -172,8 +172,9 @@ public class TestPerColumnFamilyFlush { // The total memstore size should be the same as the sum of the sizes of // memstores of CF1, CF2 and CF3. - assertEquals(totalMemstoreSize + 3 * DefaultMemStore.DEEP_OVERHEAD, cf1MemstoreSize - + cf2MemstoreSize + cf3MemstoreSize); + assertEquals( + totalMemstoreSize + (3 * (DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD)), + cf1MemstoreSize + cf2MemstoreSize + cf3MemstoreSize); // Flush! region.flush(false); @@ -192,7 +193,7 @@ public class TestPerColumnFamilyFlush { // We should have cleared out only CF1, since we chose the flush thresholds // and number of puts accordingly. - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf1MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize); // Nothing should have happened to CF2, ... assertEquals(cf2MemstoreSize, oldCF2MemstoreSize); // ... or CF3 @@ -201,8 +202,9 @@ public class TestPerColumnFamilyFlush { // LSN in the memstore of CF2. assertEquals(smallestSeqInRegionCurrentMemstore, smallestSeqCF2); // Of course, this should hold too. - assertEquals(totalMemstoreSize + 2 * DefaultMemStore.DEEP_OVERHEAD, cf2MemstoreSize - + cf3MemstoreSize); + assertEquals( + totalMemstoreSize + (2 * (DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD)), + cf2MemstoreSize + cf3MemstoreSize); // Now add more puts (mostly for CF2), so that we only flush CF2 this time. for (int i = 1200; i < 2400; i++) { @@ -229,11 +231,12 @@ public class TestPerColumnFamilyFlush { .getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); // CF1 and CF2, both should be absent. - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf1MemstoreSize); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf2MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf2MemstoreSize); // CF3 shouldn't have been touched. assertEquals(cf3MemstoreSize, oldCF3MemstoreSize); - assertEquals(totalMemstoreSize + DefaultMemStore.DEEP_OVERHEAD, cf3MemstoreSize); + assertEquals(totalMemstoreSize + (DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD), + cf3MemstoreSize); assertEquals(smallestSeqInRegionCurrentMemstore, smallestSeqCF3); // What happens when we hit the memstore limit, but we are not able to find @@ -296,8 +299,9 @@ public class TestPerColumnFamilyFlush { // The total memstore size should be the same as the sum of the sizes of // memstores of CF1, CF2 and CF3. - assertEquals(totalMemstoreSize + 3 * DefaultMemStore.DEEP_OVERHEAD, cf1MemstoreSize - + cf2MemstoreSize + cf3MemstoreSize); + assertEquals( + totalMemstoreSize + (3 * (DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD)), + cf1MemstoreSize + cf2MemstoreSize + cf3MemstoreSize); // Flush! region.flush(false); @@ -310,9 +314,9 @@ public class TestPerColumnFamilyFlush { region.getWAL().getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); // Everything should have been cleared - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf1MemstoreSize); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf2MemstoreSize); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf3MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf2MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf3MemstoreSize); assertEquals(0, totalMemstoreSize); assertEquals(HConstants.NO_SEQNUM, smallestSeqInRegionCurrentMemstore); HBaseTestingUtility.closeRegionAndWAL(region); @@ -379,12 +383,13 @@ public class TestPerColumnFamilyFlush { cf3MemstoreSize = desiredRegion.getStore(FAMILY3).getMemStoreSize(); // CF1 Should have been flushed - assertEquals(DefaultMemStore.DEEP_OVERHEAD, cf1MemstoreSize); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, cf1MemstoreSize); // CF2 and CF3 shouldn't have been flushed. assertTrue(cf2MemstoreSize > 0); assertTrue(cf3MemstoreSize > 0); - assertEquals(totalMemstoreSize + 2 * DefaultMemStore.DEEP_OVERHEAD, cf2MemstoreSize - + cf3MemstoreSize); + assertEquals( + totalMemstoreSize + (2 * (DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD)), + cf2MemstoreSize + cf3MemstoreSize); // Wait for the RS report to go across to the master, so that the master // is aware of which sequence ids have been flushed, before we kill the RS. @@ -521,12 +526,12 @@ public class TestPerColumnFamilyFlush { }); LOG.info("Finished waiting on flush after too many WALs..."); // Individual families should have been flushed. - assertEquals(DefaultMemStore.DEEP_OVERHEAD, - desiredRegion.getStore(FAMILY1).getMemStoreSize()); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, - desiredRegion.getStore(FAMILY2).getMemStoreSize()); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, - desiredRegion.getStore(FAMILY3).getMemStoreSize()); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, + desiredRegion.getStore(FAMILY1).getMemStoreSize()); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, + desiredRegion.getStore(FAMILY2).getMemStoreSize()); + assertEquals(DefaultMemStore.DEEP_OVERHEAD + MutableSegment.DEEP_OVERHEAD, + desiredRegion.getStore(FAMILY3).getMemStoreSize()); // let WAL cleanOldLogs assertNull(getWAL(desiredRegion).rollWriter(true)); assertTrue(getNumRolledLogFiles(desiredRegion) < maxLogs);
