Repository: hbase Updated Branches: refs/heads/branch-2 bfab1e2f9 -> 079e08d7c
HBASE-20390: IMC Default Parameters for 2.0.0 Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/079e08d7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/079e08d7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/079e08d7 Branch: refs/heads/branch-2 Commit: 079e08d7c5f48c14170237aaf2beffdb5e6bb8f1 Parents: bfab1e2 Author: eshcar <[email protected]> Authored: Thu May 24 18:10:10 2018 +0300 Committer: eshcar <[email protected]> Committed: Thu May 24 18:10:10 2018 +0300 ---------------------------------------------------------------------- .../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++++----- .../hbase/regionserver/MemStoreCompactionStrategy.java | 2 +- .../regionserver/TestCompactingToCellFlatMapMemStore.java | 4 +++- src/main/asciidoc/_chapters/inmemory_compaction.adoc | 7 ++++--- 4 files changed, 13 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/079e08d7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index fc3e5ba..3886e7d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -43,12 +43,12 @@ import org.apache.hadoop.hbase.wal.WAL; /** * A memstore implementation which supports in-memory compaction. * A compaction pipeline is added between the active set and the snapshot data structures; - * it consists of a list of kv-sets that are subject to compaction. - * Like the snapshot, all pipeline components are read-only; updates only affect the active set. + * it consists of a list of segments that are subject to compaction. + * Like the snapshot, all pipeline segments are read-only; updates only affect the active set. * To ensure this property we take advantage of the existing blocking mechanism -- the active set * is pushed to the pipeline while holding the region's updatesLock in exclusive mode. - * Periodically, a compaction is applied in the background to all pipeline components resulting - * in a single read-only component. The ``old'' components are discarded when no scanner is reading + * Periodically, a compaction is applied in the background to all pipeline segments resulting + * in a single read-only component. The ``old'' segments are discarded when no scanner is reading * them. */ @InterfaceAudience.Private @@ -62,7 +62,7 @@ public class CompactingMemStore extends AbstractMemStore { // Default fraction of in-memory-flush size w.r.t. flush-to-disk size public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY = "hbase.memstore.inmemoryflush.threshold.factor"; - private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.1; + private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.014; private static final Logger LOG = LoggerFactory.getLogger(CompactingMemStore.class); private HStore store; http://git-wip-us.apache.org/repos/asf/hbase/blob/079e08d7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java index 2f16055..6d28e37 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java @@ -42,7 +42,7 @@ public abstract class MemStoreCompactionStrategy { // The upper bound for the number of segments we store in the pipeline prior to merging. public static final String COMPACTING_MEMSTORE_THRESHOLD_KEY = "hbase.hregion.compacting.pipeline.segments.limit"; - public static final int COMPACTING_MEMSTORE_THRESHOLD_DEFAULT = 4; + public static final int COMPACTING_MEMSTORE_THRESHOLD_DEFAULT = 2; /** * Types of actions to be done on the pipeline upon MemStoreCompaction invocation. http://git-wip-us.apache.org/repos/asf/hbase/blob/079e08d7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index bea8c4f..70d6970 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -848,8 +848,10 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore // set memstore to flat into CellChunkMap MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; + memstore.getConfiguration().setInt(MemStoreCompactionStrategy + .COMPACTING_MEMSTORE_THRESHOLD_KEY, 4); memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(compactionType)); + String.valueOf(compactionType)); ((MyCompactingMemStore) memstore).initiateType(compactionType, memstore.getConfiguration()); ((CompactingMemStore) memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); http://git-wip-us.apache.org/repos/asf/hbase/blob/079e08d7/src/main/asciidoc/_chapters/inmemory_compaction.adoc ---------------------------------------------------------------------- diff --git a/src/main/asciidoc/_chapters/inmemory_compaction.adoc b/src/main/asciidoc/_chapters/inmemory_compaction.adoc index da3abb1..21513f3 100644 --- a/src/main/asciidoc/_chapters/inmemory_compaction.adoc +++ b/src/main/asciidoc/_chapters/inmemory_compaction.adoc @@ -52,7 +52,7 @@ In this section we describe how to enable Accordion and the available configurat To enable in-memory compactions, set the _IN_MEMORY_COMPACTION_ attribute on per column family where you want the behavior. The _IN_MEMORY_COMPACTION_ -attribute can have one of three values. +attribute can have one of four values. * _NONE_: No in-memory compaction. * _BASIC_: Basic policy enables flushing and keeps a pipeline of flushes until we trip the pipeline maximum threshold and then we flush to disk. No in-memory compaction but can help throughput as data is moved from the profligate, native ConcurrentSkipListMap data-type to more compact (and efficient) data types. @@ -94,11 +94,12 @@ to set the in-memory attribute to _NONE_). When an in-memory flush happens is calculated by dividing the configured region flush size (Set in the table descriptor or read from _hbase.hregion.memstore.flush.size_) by the number of column families and then multiplying by -_hbase.memstore.inmemoryflush.threshold.factor_ (default is 0.1). +_hbase.memstore.inmemoryflush.threshold.factor_ (default is 0.014 - a little less than the size +of a chunk). The number of flushes carried by the pipeline is monitored so as to fit within the bounds of memstore sizing but you can also set a maximum on the number of flushes total by setting -_hbase.hregion.compacting.pipeline.segments.limit_. Default is 4. +_hbase.hregion.compacting.pipeline.segments.limit_. Default is 2. When a column family Store is created, it says what memstore type is in effect. As of this writing there is the old-school _DefaultMemStore_ which fills a _ConcurrentSkipListMap_ and then flushes
