Repository: hbase Updated Branches: refs/heads/branch-2 b4e3798bb -> 0907563eb
HBASE-19282: Making CellChunkMap the default index (CellSet delegatee) for ImmutableSegments, when MSLAB is used. In order to avoid additional user settings. If no MSLAB is requested the index is going to be CellArrayMap Signed-off-by: Anastasia Braginsky <anas...@yahoo-inc.com> Signed-off-by: Michael Stack <st...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0907563e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0907563e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0907563e Branch: refs/heads/branch-2 Commit: 0907563eb72697b394b8b960fe54887d6ff304fd Parents: b4e3798 Author: anastas <anas...@yahoo-inc.com> Authored: Thu Dec 28 22:44:06 2017 +0200 Committer: Michael Stack <st...@apache.org> Committed: Thu Dec 28 16:49:56 2017 -0800 ---------------------------------------------------------------------- .../hbase/regionserver/CompactingMemStore.java | 45 ++++++++++++-------- .../hbase/regionserver/MemStoreLABImpl.java | 18 +++++--- .../regionserver/TestCompactingMemStore.java | 1 + .../TestCompactingToCellFlatMapMemStore.java | 45 ++++++++------------ .../TestWalAndCompactingMemStoreFlush.java | 9 +++- 5 files changed, 66 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0907563e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 94cc0b4..d874b2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -58,16 +58,10 @@ public class CompactingMemStore extends AbstractMemStore { "hbase.hregion.compacting.memstore.type"; public static final String COMPACTING_MEMSTORE_TYPE_DEFAULT = String.valueOf(MemoryCompactionPolicy.BASIC); - // The external setting of the compacting MemStore behaviour - public static final String COMPACTING_MEMSTORE_INDEX_KEY = - "hbase.hregion.compacting.memstore.index"; - // usage of CellArrayMap is default, later it will be decided how to use CellChunkMap - public static final String COMPACTING_MEMSTORE_INDEX_DEFAULT = - String.valueOf(IndexType.ARRAY_MAP); // Default fraction of in-memory-flush size w.r.t. flush-to-disk size public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY = "hbase.memstore.inmemoryflush.threshold.factor"; - private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.02; + private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.1; private static final Logger LOG = LoggerFactory.getLogger(CompactingMemStore.class); private HStore store; @@ -114,9 +108,16 @@ public class CompactingMemStore extends AbstractMemStore { this.regionServices = regionServices; this.pipeline = new CompactionPipeline(getRegionServices()); this.compactor = createMemStoreCompactor(compactionPolicy); + if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) { + // if user requested to work with MSLABs (whether on- or off-heap), then the + // immutable segments are going to use CellChunkMap as their index + indexType = IndexType.CHUNK_MAP; + } else { + indexType = IndexType.ARRAY_MAP; + } + // initialization of the flush size should happen after initialization of the index type + // so do not transfer the following method initInmemoryFlushSize(conf); - indexType = IndexType.valueOf(conf.get(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT)); } @VisibleForTesting @@ -126,6 +127,7 @@ public class CompactingMemStore extends AbstractMemStore { } private void initInmemoryFlushSize(Configuration conf) { + double factor = 0; long memstoreFlushSize = getRegionServices().getMemStoreFlushSize(); int numStores = getRegionServices().getNumStores(); if (numStores <= 1) { @@ -133,11 +135,17 @@ public class CompactingMemStore extends AbstractMemStore { numStores = 1; } inmemoryFlushSize = memstoreFlushSize / numStores; - // multiply by a factor - double factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, - IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); + // multiply by a factor (different factors for different index types) + if (indexType == IndexType.ARRAY_MAP) { + factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, + IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); + } else { + factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, + IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); + } inmemoryFlushSize *= factor; - LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize); + LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize + + " and immutable segments index to be of type " + indexType); } /** @@ -318,10 +326,11 @@ public class CompactingMemStore extends AbstractMemStore { // setter is used only for testability @VisibleForTesting - public void setIndexType() { - indexType = IndexType.valueOf(getConfiguration().get( - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT)); + void setIndexType(IndexType type) { + indexType = type; + // Because this functionality is for testing only and tests are setting in-memory flush size + // according to their need, there is no setting of in-memory flush size, here. + // If it is needed, please change in-memory flush size explicitly } public IndexType getIndexType() { @@ -572,7 +581,7 @@ public class CompactingMemStore extends AbstractMemStore { // debug method public void debug() { String msg = "active size=" + this.active.keySize(); - msg += " threshold="+IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT* inmemoryFlushSize; + msg += " in-memory flush size is "+ inmemoryFlushSize; msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false"); msg += " inMemoryFlushInProgress is "+ (inMemoryFlushInProgress.get() ? "true" : "false"); LOG.debug(msg); http://git-wip-us.apache.org/repos/asf/hbase/blob/0907563e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index 6545009..f7728ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -77,6 +77,7 @@ public class MemStoreLABImpl implements MemStoreLAB { private final int chunkSize; private final int maxAlloc; private final ChunkCreator chunkCreator; + private final CompactingMemStore.IndexType idxType; // what index is used for corresponding segment // This flag is for closing this instance, its set when clearing snapshot of // memstore @@ -99,6 +100,10 @@ public class MemStoreLABImpl implements MemStoreLAB { // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one! Preconditions.checkArgument(maxAlloc <= chunkSize, MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); + + // if user requested to work with MSLABs (whether on- or off-heap), then the + // immutable segments are going to use CellChunkMap as their index + idxType = CompactingMemStore.IndexType.CHUNK_MAP; } @Override @@ -264,7 +269,7 @@ public class MemStoreLABImpl implements MemStoreLAB { if (c != null) { return c; } - c = this.chunkCreator.getChunk(); + c = this.chunkCreator.getChunk(idxType); if (c != null) { // set the curChunk. No need of CAS as only one thread will be here curChunk.set(c); @@ -278,12 +283,15 @@ public class MemStoreLABImpl implements MemStoreLAB { return null; } - // Returning a new chunk, without replacing current chunk, - // meaning MSLABImpl does not make the returned chunk as CurChunk. - // The space on this chunk will be allocated externally - // The interface is only for external callers + /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array. + ** Returning a new chunk, without replacing current chunk, + ** meaning MSLABImpl does not make the returned chunk as CurChunk. + ** The space on this chunk will be allocated externally. + ** The interface is only for external callers + */ @Override public Chunk getNewExternalChunk() { + // the new chunk is going to be part of the chunk array and will always be referenced Chunk c = this.chunkCreator.getChunk(); chunks.add(c.getId()); return c; http://git-wip-us.apache.org/repos/asf/hbase/blob/0907563e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index ed5eee8..c0ba621 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -89,6 +89,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { compactingSetUp(); this.memstore = new MyCompactingMemStore(HBaseConfiguration.create(), CellComparator.getInstance(), store, regionServicesForStores, MemoryCompactionPolicy.EAGER); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } protected void compactingSetUp() throws Exception { http://git-wip-us.apache.org/repos/asf/hbase/blob/0907563e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index 475f687..883b649 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.util.Threads; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -94,9 +95,9 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4 if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); + } else { + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } // test 1 bucket @@ -140,9 +141,9 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testCompaction2Buckets() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); + } else { + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } String[] keys1 = { "A", "A", "B", "C" }; String[] keys2 = { "A", "B", "D" }; @@ -202,9 +203,10 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testCompaction3Buckets() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); + } else { + // set to CellArrayMap as CCM is configured by default due to MSLAB usage + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } String[] keys1 = { "A", "A", "B", "C" }; String[] keys2 = { "A", "B", "D" }; @@ -290,9 +292,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testMerging() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); } String[] keys1 = { "A", "A", "B", "C", "F", "H"}; String[] keys2 = { "A", "B", "D", "G", "I", "J"}; @@ -362,9 +362,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testTimeRangeAfterCompaction() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); } testTimeRange(true); } @@ -373,9 +371,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testTimeRangeAfterMerge() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); } MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, @@ -622,9 +618,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compactionType)); ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); - memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); int numOfCells = 8; String[] keys1 = { "A", "A", "B", "C", "D", "D", "E", "F" }; //A1, A2, B3, C4, D5, D6, E7, F8 @@ -686,9 +680,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compactionType)); ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); - memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); int numOfCells = 4; char[] chars = new char[MemStoreLAB.MAX_ALLOC_DEFAULT]; for (int i = 0; i < chars.length; i++) { @@ -749,6 +741,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore * testFlatteningToJumboCellChunkMap checks that the process of flattening * into CellChunkMap succeeds, even when such big cells are allocated. */ + @Ignore @Test public void testFlatteningToJumboCellChunkMap() throws IOException { @@ -760,9 +753,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compactionType)); ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); - memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); int numOfCells = 1; char[] chars = new char[MemStoreLAB.CHUNK_SIZE_DEFAULT]; for (int i = 0; i < chars.length; i++) { http://git-wip-us.apache.org/repos/asf/hbase/blob/0907563e/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java index 6a64796..f3bd7ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java @@ -768,8 +768,7 @@ public class TestWalAndCompactingMemStoreFlush { conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 300 * 1024); conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushNonSloppyStoresFirstPolicy.class.getName()); - conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN, - 75 * 1024); + conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN, 75 * 1024); conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.8); // set memstore to do index compaction with merge conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, @@ -797,6 +796,12 @@ public class TestWalAndCompactingMemStoreFlush { long totalMemstoreSize = region.getMemStoreSize(); + // test in-memory flashing into CAM here + ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).setIndexType( + CompactingMemStore.IndexType.ARRAY_MAP); + ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).setIndexType( + CompactingMemStore.IndexType.ARRAY_MAP); + // Find the sizes of the memstores of each CF. MemStoreSize cf1MemstoreSizePhaseI = region.getStore(FAMILY1).getMemStoreSize(); MemStoreSize cf2MemstoreSizePhaseI = region.getStore(FAMILY2).getMemStoreSize();
