hbase git commit: HBASE-17887 Row-level consistency is broken for read

2017-05-12 Thread chia7712 
Repository: hbase
Updated Branches:
  refs/heads/master 5e046151d -> b34ab5980


HBASE-17887 Row-level consistency is broken for read


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b34ab598
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b34ab598
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b34ab598

Branch: refs/heads/master
Commit: b34ab5980ea7a21fd750537476027f9a8665eacc
Parents: 5e04615
Author: Chia-Ping Tsai 
Authored: Fri May 12 19:45:07 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Fri May 12 19:45:07 2017 +0800

--
 .../regionserver/ChangedReadersObserver.java|  12 +-
 .../hbase/regionserver/CompactingMemStore.java  |  12 +-
 .../hadoop/hbase/regionserver/HStore.java   |  26 ++-
 .../hadoop/hbase/regionserver/StoreScanner.java |  38 +++-
 .../hadoop/hbase/regionserver/TestStore.java| 225 ++-
 .../hbase/regionserver/TestStoreScanner.java|   5 +-
 .../hbase/regionserver/TestWideScanner.java |   3 +-
 7 files changed, 293 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b34ab598/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
index 0bc75e7..a019666 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
@@ -21,18 +21,24 @@ package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
 import java.util.List;
-
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 
-
 /**
  * If set of MapFile.Readers in Store change, implementors are notified.
  */
 @InterfaceAudience.Private
 public interface ChangedReadersObserver {
+
+  /**
+   * @return the read point of the current scan
+   */
+  long getReadPoint();
+
   /**
* Notify observers.
+   * @param sfs The new files
+   * @param memStoreScanners scanner of current memstore
* @throws IOException e
*/
-  void updateReaders(List sfs) throws IOException;
+  void updateReaders(List sfs, List 
memStoreScanners) throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b34ab598/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index aaf60e3..8d71efc 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -317,18 +317,24 @@ public class CompactingMemStore extends AbstractMemStore {
* Scanners are ordered from 0 (oldest) to newest in increasing order.
*/
   public List getScanners(long readPt) throws IOException {
+MutableSegment activeTmp = active;
 List pipelineList = pipeline.getSegments();
 List snapshotList = snapshot.getAllSegments();
 long order = 1 + pipelineList.size() + snapshotList.size();
 // The list of elements in pipeline + the active element + the snapshot 
segment
 // The order is the Segment ordinal
-List list = new ArrayList((int) order);
-order = addToScanners(active, readPt, order, list);
+List list = createList((int) order);
+order = addToScanners(activeTmp, readPt, order, list);
 order = addToScanners(pipelineList, readPt, order, list);
 addToScanners(snapshotList, readPt, order, list);
 return list;
   }
 
+   @VisibleForTesting
+   protected List createList(int capacity) {
+ return new ArrayList<>(capacity);
+   }
+
   /**
* Check whether anything need to be done based on the current active set 
size.
* The method is invoked upon every addition to the active set.
@@ -428,7 +434,7 @@ public class CompactingMemStore extends AbstractMemStore {
 }
   }
 
-  private void pushActiveToPipeline(MutableSegment active) {
+  protected void pushActiveToPipeline(MutableSegment active) {
 if (!active.isEmpty()) {
   pipeline.pushHead(active);
   resetActive();

http://git-wip-us.apache.org/repos/asf/hbase/blob/b34ab598/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
--
diff --git

hbase git commit: HBASE-17887 Row-level consistency is broken for read

2017-05-12 Thread chia7712 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 51cb53776 -> f81486445


HBASE-17887 Row-level consistency is broken for read


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f8148644
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f8148644
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f8148644

Branch: refs/heads/branch-1
Commit: f81486445c072096022cca77eb0a53f1594ff204
Parents: 51cb537
Author: Chia-Ping Tsai 
Authored: Fri May 12 19:44:16 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Fri May 12 19:44:16 2017 +0800

--
 .../regionserver/ChangedReadersObserver.java|   9 +-
 .../hbase/regionserver/DefaultMemStore.java | 301 ++-
 .../hadoop/hbase/regionserver/HStore.java   |   9 +-
 .../hadoop/hbase/regionserver/StoreScanner.java |  45 ++-
 .../apache/hadoop/hbase/io/TestHeapSize.java|   3 +-
 .../hbase/regionserver/TestDefaultMemStore.java |  68 ++---
 .../hadoop/hbase/regionserver/TestHRegion.java  |   4 +-
 .../regionserver/TestMemStoreChunkPool.java |  12 +-
 .../hadoop/hbase/regionserver/TestStore.java| 155 +-
 .../hbase/regionserver/TestStoreScanner.java|   5 +-
 .../hbase/regionserver/TestWideScanner.java |   3 +-
 11 files changed, 401 insertions(+), 213 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f8148644/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
index 0bc75e7..4421ac5 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
@@ -30,9 +30,16 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
  */
 @InterfaceAudience.Private
 public interface ChangedReadersObserver {
+
+  /**
+   * @return the read point of the current scan
+   */
+  long getReadPoint();
   /**
* Notify observers.
+   * @param sfs The new files
+   * @param memStoreScanners scanner of current memstore
* @throws IOException e
*/
-  void updateReaders(List sfs) throws IOException;
+  void updateReaders(List sfs, List 
memStoreScanners) throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8148644/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 7b7446a..b1e9f32 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -71,38 +71,26 @@ import com.google.common.annotations.VisibleForTesting;
 @InterfaceAudience.Private
 public class DefaultMemStore implements MemStore {
   private static final Log LOG = LogFactory.getLog(DefaultMemStore.class);
+  @VisibleForTesting
   static final String USEMSLAB_KEY = "hbase.hregion.memstore.mslab.enabled";
   private static final boolean USEMSLAB_DEFAULT = true;
-  static final String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class";
+  private static final String MSLAB_CLASS_NAME = 
"hbase.regionserver.mslab.class";
 
   private Configuration conf;
 
-  // MemStore.  Use a CellSkipListSet rather than SkipListSet because of the
-  // better semantics.  The Map will overwrite if passed a key it already had
-  // whereas the Set will not add new Cell if key is same though value might be
-  // different.  Value is not important -- just make sure always same
-  // reference passed.
-  volatile CellSkipListSet cellSet;
-
-  // Snapshot of memstore.  Made for flusher.
-  volatile CellSkipListSet snapshot;
-
+  @VisibleForTesting
   final KeyValue.KVComparator comparator;
 
-  // Used to track own heapSize
-  final AtomicLong size;
-  private volatile long snapshotSize;
-
   // Used to track when to flush
-  volatile long timeOfOldestEdit = Long.MAX_VALUE;
+  private volatile long timeOfOldestEdit = Long.MAX_VALUE;
 
-  TimeRangeTracker timeRangeTracker;
-  TimeRangeTracker snapshotTimeRangeTracker;
+  private volatile long snapshotId;
+  private volatile boolean tagsPresent;
 
-  volatile MemStoreLAB allocator;
-  volatile MemStoreLAB snapshotAllocator;
-  volatile long snapshotId;
-  volatile boolean

hbase git commit: HBASE-17887 Row-level consistency is broken for read

2017-05-12 Thread chia7712 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 5cbc041b9 -> 72edf521c


HBASE-17887 Row-level consistency is broken for read


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72edf521
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72edf521
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72edf521

Branch: refs/heads/branch-1.3
Commit: 72edf521c1effe3afe6ce6b39aaf843b8651a4a6
Parents: 5cbc041
Author: Chia-Ping Tsai 
Authored: Fri May 12 19:42:45 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Fri May 12 19:42:45 2017 +0800

--
 .../regionserver/ChangedReadersObserver.java|   9 +-
 .../hbase/regionserver/DefaultMemStore.java | 300 ++-
 .../hadoop/hbase/regionserver/HStore.java   |   9 +-
 .../hadoop/hbase/regionserver/StoreScanner.java |  44 ++-
 .../apache/hadoop/hbase/io/TestHeapSize.java|   3 +-
 .../hbase/regionserver/TestDefaultMemStore.java |  66 ++--
 .../hadoop/hbase/regionserver/TestHRegion.java  |   4 +-
 .../regionserver/TestMemStoreChunkPool.java |  12 +-
 .../hadoop/hbase/regionserver/TestStore.java| 151 +-
 .../hbase/regionserver/TestStoreScanner.java|   5 +-
 .../hbase/regionserver/TestWideScanner.java |   3 +-
 11 files changed, 398 insertions(+), 208 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/72edf521/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
index 0bc75e7..4421ac5 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.java
@@ -30,9 +30,16 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
  */
 @InterfaceAudience.Private
 public interface ChangedReadersObserver {
+
+  /**
+   * @return the read point of the current scan
+   */
+  long getReadPoint();
   /**
* Notify observers.
+   * @param sfs The new files
+   * @param memStoreScanners scanner of current memstore
* @throws IOException e
*/
-  void updateReaders(List sfs) throws IOException;
+  void updateReaders(List sfs, List 
memStoreScanners) throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/72edf521/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 70f5a12..884ef29 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -70,38 +70,26 @@ import com.google.common.annotations.VisibleForTesting;
 @InterfaceAudience.Private
 public class DefaultMemStore implements MemStore {
   private static final Log LOG = LogFactory.getLog(DefaultMemStore.class);
+  @VisibleForTesting
   static final String USEMSLAB_KEY = "hbase.hregion.memstore.mslab.enabled";
   private static final boolean USEMSLAB_DEFAULT = true;
-  static final String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class";
+  private static final String MSLAB_CLASS_NAME = 
"hbase.regionserver.mslab.class";
 
   private Configuration conf;
 
-  // MemStore.  Use a CellSkipListSet rather than SkipListSet because of the
-  // better semantics.  The Map will overwrite if passed a key it already had
-  // whereas the Set will not add new Cell if key is same though value might be
-  // different.  Value is not important -- just make sure always same
-  // reference passed.
-  volatile CellSkipListSet cellSet;
-
-  // Snapshot of memstore.  Made for flusher.
-  volatile CellSkipListSet snapshot;
-
+  @VisibleForTesting
   final KeyValue.KVComparator comparator;
 
-  // Used to track own heapSize
-  final AtomicLong size;
-  private volatile long snapshotSize;
-
   // Used to track when to flush
-  volatile long timeOfOldestEdit = Long.MAX_VALUE;
+  private volatile long timeOfOldestEdit = Long.MAX_VALUE;
 
-  TimeRangeTracker timeRangeTracker;
-  TimeRangeTracker snapshotTimeRangeTracker;
+  private volatile long snapshotId;
+  private volatile boolean tagsPresent;
 
-  volatile MemStoreLAB allocator;
-  volatile MemStoreLAB snapshotAllocator;
-  volatile long snapshotId;
-  volatile boolean