hbase git commit: HBASE-17434: New synchronization scheme for compaction pipeline

2017-01-12 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/master f7d0f15c9 -> 2f8ddf6fc


HBASE-17434: New synchronization scheme for compaction pipeline

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2f8ddf6f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2f8ddf6f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2f8ddf6f

Branch: refs/heads/master
Commit: 2f8ddf6fc5f904f0273b07469286e01aa02c7da5
Parents: f7d0f15
Author: eshcar 
Authored: Sun Jan 8 22:30:44 2017 +0200
Committer: Michael Stack 
Committed: Thu Jan 12 06:35:58 2017 -0800

--
 .../hbase/regionserver/CompactingMemStore.java  |  6 +-
 .../hbase/regionserver/CompactionPipeline.java  | 78 
 .../apache/hadoop/hbase/io/TestHeapSize.java|  2 +
 3 files changed, 53 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2f8ddf6f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index e1289f8..99c1685 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -217,8 +217,8 @@ public class CompactingMemStore extends AbstractMemStore {
   @VisibleForTesting
   @Override
   protected List getSegments() {
-List pipelineList = pipeline.getSegments();
-List list = new ArrayList(pipelineList.size() + 2);
+List pipelineList = pipeline.getSegments();
+List list = new ArrayList<>(pipelineList.size() + 2);
 list.add(this.active);
 list.addAll(pipelineList);
 list.add(this.snapshot);
@@ -264,7 +264,7 @@ public class CompactingMemStore extends AbstractMemStore {
* Scanners are ordered from 0 (oldest) to newest in increasing order.
*/
   public List getScanners(long readPt) throws IOException {
-List pipelineList = pipeline.getSegments();
+List pipelineList = pipeline.getSegments();
 long order = pipelineList.size();
 // The list of elements in pipeline + the active element + the snapshot 
segment
 // TODO : This will change when the snapshot is made of more than one 
element

http://git-wip-us.apache.org/repos/asf/hbase/blob/2f8ddf6f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
index 9d5df77..fafdbee 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
@@ -25,50 +25,65 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.CellComparator;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 
 /**
  * The compaction pipeline of a {@link CompactingMemStore}, is a FIFO queue of 
segments.
- * It supports pushing a segment at the head of the pipeline and pulling a 
segment from the
- * tail to flush to disk.
- * It also supports swap operation to allow the compactor swap a subset of the 
segments with a new
- * (compacted) one. This swap succeeds only if the version number passed with 
the list of segments
- * to swap is the same as the current version of the pipeline.
- * The pipeline version is updated whenever swapping segments or pulling the 
segment at the tail.
+ * It supports pushing a segment at the head of the pipeline and removing a 
segment from the
+ * tail when it is flushed to disk.
+ * It also supports swap method to allow the in-memory compaction swap a 
subset of the segments
+ * at the tail of the pipeline with a new (compacted) one. This swap succeeds 
only if the version
+ * number passed with the list of segments to swap is the same as the current 
version of the
+ * pipeline.
+ * Essentially, there are two methods which can change the structure of the 
pipeline: pushHead()
+ * and swap(), the later is used both by a flush to disk and by an in-memory 
compaction.
+ * The pipeline version is updated by swap(); it allows to identify 
conflicting operations at the
+ *

[1/2] hbase git commit: HBASE-17434 New Synchronization Scheme for Compaction Pipeline (Eshcar Hillel)

2017-01-09 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/master 9cbeba6c3 -> dd1ae3714


HBASE-17434 New Synchronization Scheme for Compaction Pipeline (Eshcar Hillel)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15762691
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15762691
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15762691

Branch: refs/heads/master
Commit: 1576269123f18c9eb21b04a800e81952ec52c04d
Parents: 9cbeba6
Author: Michael Stack 
Authored: Mon Jan 9 10:46:34 2017 -0800
Committer: Michael Stack 
Committed: Mon Jan 9 10:46:34 2017 -0800

--
 .../hbase/regionserver/CompactingMemStore.java  |  6 +-
 .../hbase/regionserver/CompactionPipeline.java  | 76 
 .../apache/hadoop/hbase/io/TestHeapSize.java|  2 +
 3 files changed, 51 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/15762691/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index e1289f8..99c1685 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -217,8 +217,8 @@ public class CompactingMemStore extends AbstractMemStore {
   @VisibleForTesting
   @Override
   protected List getSegments() {
-List pipelineList = pipeline.getSegments();
-List list = new ArrayList(pipelineList.size() + 2);
+List pipelineList = pipeline.getSegments();
+List list = new ArrayList<>(pipelineList.size() + 2);
 list.add(this.active);
 list.addAll(pipelineList);
 list.add(this.snapshot);
@@ -264,7 +264,7 @@ public class CompactingMemStore extends AbstractMemStore {
* Scanners are ordered from 0 (oldest) to newest in increasing order.
*/
   public List getScanners(long readPt) throws IOException {
-List pipelineList = pipeline.getSegments();
+List pipelineList = pipeline.getSegments();
 long order = pipelineList.size();
 // The list of elements in pipeline + the active element + the snapshot 
segment
 // TODO : This will change when the snapshot is made of more than one 
element

http://git-wip-us.apache.org/repos/asf/hbase/blob/15762691/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
index 9d5df77..ebc8c4b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
@@ -25,50 +25,63 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.CellComparator;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 
 /**
  * The compaction pipeline of a {@link CompactingMemStore}, is a FIFO queue of 
segments.
- * It supports pushing a segment at the head of the pipeline and pulling a 
segment from the
- * tail to flush to disk.
- * It also supports swap operation to allow the compactor swap a subset of the 
segments with a new
- * (compacted) one. This swap succeeds only if the version number passed with 
the list of segments
- * to swap is the same as the current version of the pipeline.
- * The pipeline version is updated whenever swapping segments or pulling the 
segment at the tail.
+ * It supports pushing a segment at the head of the pipeline and removing a 
segment from the
+ * tail when it is flushed to disk.
+ * It also supports swap method to allow the in-memory compaction swap a 
subset of the segments
+ * at the tail of the pipeline with a new (compacted) one. This swap succeeds 
only if the version
+ * number passed with the list of segments to swap is the same as the current 
version of the
+ * pipeline.
+ * Essentially, there are two methods which can change the structure of the 
pipeline: pushHead()
+ * and swap(), the later is used both by a flush to disk and by an in-memory 
compaction.
+ * The pipeline version is updated by swap(); it allows to identify 
conflicting operations at the
+ * suffix of the pipeline.
+ *
+ *