hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM 2 for making sure that MVCC read point is advanced after flush

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 ba6e7ddd3 -> 382f88ae8


HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM 2 for 
making sure that MVCC read point is advanced after flush


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/382f88ae
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/382f88ae
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/382f88ae

Branch: refs/heads/branch-1.1
Commit: 382f88ae87490cf0860aa63e104bf7e8273827c4
Parents: ba6e7dd
Author: Enis Soztutar 
Authored: Mon Oct 17 14:30:01 2016 -0700
Committer: Enis Soztutar 
Committed: Mon Oct 17 14:30:01 2016 -0700

--
 .../hadoop/hbase/regionserver/HRegion.java  | 20 +++-
 1 file changed, 11 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/382f88ae/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index cc89b84..e600ae1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2198,16 +2198,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 long trxId = 0;
 try {
   try {
+mvcc.waitForPreviousTransactionsComplete();
 writeEntry = mvcc.beginMemstoreInsert();
-// wait for all in-progress transactions to commit to WAL before   

   
-// we can start the flush. This prevents
-// uncommitted transactions from being written into HFiles.
-// We have to block before we start the flush, otherwise keys that
-// were removed via a rollbackMemstore could be written to Hfiles.
-mvcc.waitForPreviousTransactionsComplete(writeEntry);
-// set w to null to prevent mvcc.advanceMemstore from being called 
again inside finally block
-writeEntry = null;
-
 if (wal != null) {
   Long earliestUnflushedSequenceIdForTheRegion =
   wal.startCacheFlush(encodedRegionName, flushedFamilyNames);
@@ -2284,6 +2276,16 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   throw ioe;
 }
   }
+
+  // wait for all in-progress transactions to commit to WAL before
+  // we can start the flush. This prevents
+  // uncommitted transactions from being written into HFiles.
+  // We have to block before we start the flush, otherwise keys that
+  // were removed via a rollbackMemstore could be written to Hfiles.
+  writeEntry.setWriteNumber(flushOpSeqId);
+  mvcc.waitForPreviousTransactionsComplete(writeEntry);
+  // set w to null to prevent mvcc.advanceMemstore from being called again 
inside finally block
+  writeEntry = null;
 } finally {
   if (writeEntry != null) {
 // in case of failure just mark current writeEntry as complete

[3/5] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to change WAL to extend Closeable

[enis]

HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to 
change WAL to extend Closeable


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3ddff3b6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3ddff3b6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3ddff3b6

Branch: refs/heads/branch-1.3
Commit: 3ddff3b665dbf83f138f8ab19b4e79f391ff71a8
Parents: 9f36408
Author: Enis Soztutar 
Authored: Sat Oct 1 14:50:49 2016 -0700
Committer: Enis Soztutar 
Committed: Sat Oct 1 14:52:38 2016 -0700

--
 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3ddff3b6/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 041a5b9..413cbaa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -47,7 +47,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL extends AutoCloseable {
+public interface WAL extends Closeable {
 
   /**
* Registers WALActionsListener

[2/5] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to change WAL to extend Closeable

[enis]

HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to 
change WAL to extend Closeable


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/06cc1238
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/06cc1238
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/06cc1238

Branch: refs/heads/branch-1
Commit: 06cc123849aefb67570f0c016829b53ab958721b
Parents: 911f9b9
Author: Enis Soztutar 
Authored: Sat Oct 1 14:50:49 2016 -0700
Committer: Enis Soztutar 
Committed: Sat Oct 1 14:50:54 2016 -0700

--
 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/06cc1238/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index e43d44f..9e90a0c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -51,7 +51,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL extends AutoCloseable {
+public interface WAL extends Closeable {
 
   /**
* Registers WALActionsListener

[4/5] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to change WAL to extend Closeable

[enis]

HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to 
change WAL to extend Closeable


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77e25d32
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77e25d32
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77e25d32

Branch: refs/heads/branch-1.2
Commit: 77e25d32b3ad8863625c9d25e3ecd7526608acf6
Parents: 2e381ee
Author: Enis Soztutar 
Authored: Sat Oct 1 14:50:49 2016 -0700
Committer: Enis Soztutar 
Committed: Sat Oct 1 14:54:26 2016 -0700

--
 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/77e25d32/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 041a5b9..413cbaa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -47,7 +47,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL extends AutoCloseable {
+public interface WAL extends Closeable {
 
   /**
* Registers WALActionsListener

[5/5] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to change WAL to extend Closeable

[enis]

HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to 
change WAL to extend Closeable


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8019307
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8019307
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8019307

Branch: refs/heads/branch-1.1
Commit: e8019307c58da488ad26555d104990cd1c005dab
Parents: 88bf5b3
Author: Enis Soztutar 
Authored: Sat Oct 1 14:50:49 2016 -0700
Committer: Enis Soztutar 
Committed: Sat Oct 1 14:55:55 2016 -0700

--
 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e8019307/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 20d0834..4a73895 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -50,7 +50,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL extends AutoCloseable {
+public interface WAL extends Closeable {
 
   /**
* Registers WALActionsListener

[1/5] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to change WAL to extend Closeable

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 911f9b9eb -> 06cc12384
  refs/heads/branch-1.1 88bf5b3b1 -> e8019307c
  refs/heads/branch-1.2 2e381ee2d -> 77e25d32b
  refs/heads/branch-1.3 9f364084a -> 3ddff3b66
  refs/heads/master c3c82f355 -> b8ad9b17b


HBASE-16721 Concurrency issue in WAL unflushed seqId tracking - ADDENDUM to 
change WAL to extend Closeable


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b8ad9b17
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b8ad9b17
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b8ad9b17

Branch: refs/heads/master
Commit: b8ad9b17bb6365026036687a47b3586958596d68
Parents: c3c82f3
Author: Enis Soztutar 
Authored: Sat Oct 1 14:50:49 2016 -0700
Committer: Enis Soztutar 
Committed: Sat Oct 1 14:50:49 2016 -0700

--
 hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b8ad9b17/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 20ae602..c38c262 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.hbase.util.Bytes;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL extends AutoCloseable {
+public interface WAL extends Closeable {
 
   /**
* Registers WALActionsListener

[2/3] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking

[enis]

HBASE-16721 Concurrency issue in WAL unflushed seqId tracking


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f77f1530
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f77f1530
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f77f1530

Branch: refs/heads/branch-1.3
Commit: f77f1530d4cebd1679bc1c27782bc283638dbd5f
Parents: 728f58a
Author: Enis Soztutar 
Authored: Thu Sep 29 13:50:58 2016 -0700
Committer: Enis Soztutar 
Committed: Thu Sep 29 14:53:29 2016 -0700

--
 .../hadoop/hbase/regionserver/HRegion.java  |  19 ++--
 .../java/org/apache/hadoop/hbase/wal/WAL.java   |   2 +-
 .../hbase/regionserver/wal/TestFSHLog.java  | 101 ++-
 3 files changed, 110 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f77f1530/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index d43e838..520286f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2293,6 +2293,15 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 
 long trxId = 0;
 MultiVersionConcurrencyControl.WriteEntry writeEntry = mvcc.begin();
+// wait for all in-progress transactions to commit to WAL before
+// we can start the flush. This prevents
+// uncommitted transactions from being written into HFiles.
+// We have to block before we start the flush, otherwise keys that
+// were removed via a rollbackMemstore could be written to Hfiles.
+mvcc.completeAndWait(writeEntry);
+// set writeEntry to null to prevent mvcc.complete from being called again 
inside finally
+// block
+writeEntry = null;
 try {
   try {
 if (wal != null) {
@@ -2371,16 +2380,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   throw ioe;
 }
   }
-
-  // wait for all in-progress transactions to commit to WAL before
-  // we can start the flush. This prevents
-  // uncommitted transactions from being written into HFiles.
-  // We have to block before we start the flush, otherwise keys that
-  // were removed via a rollbackMemstore could be written to Hfiles.
-  mvcc.completeAndWait(writeEntry);
-  // set writeEntry to null to prevent mvcc.complete from being called 
again inside finally
-  // block
-  writeEntry = null;
 } finally {
   if (writeEntry != null) {
 // In case of failure just mark current writeEntry as complete.

http://git-wip-us.apache.org/repos/asf/hbase/blob/f77f1530/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index d2b336e..041a5b9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -47,7 +47,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL {
+public interface WAL extends AutoCloseable {
 
   /**
* Registers WALActionsListener

http://git-wip-us.apache.org/repos/asf/hbase/blob/f77f1530/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index 6ece700..760cdc1 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -30,6 +30,10 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.lang.mutable.MutableBoolean;
 import org.apache.commons.logging.Log;
@@ -55,6 +59,7 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import 

[1/3] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 5ac2776d2 -> bf5a7aba5
  refs/heads/branch-1.2 42dff8a58 -> cf374af10
  refs/heads/branch-1.3 728f58ad5 -> f77f1530d


HBASE-16721 Concurrency issue in WAL unflushed seqId tracking


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bf5a7aba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bf5a7aba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bf5a7aba

Branch: refs/heads/branch-1
Commit: bf5a7aba5c0c83874f52cbd775dd280cb4a1cd49
Parents: 5ac2776
Author: Enis Soztutar 
Authored: Thu Sep 29 13:50:58 2016 -0700
Committer: Enis Soztutar 
Committed: Thu Sep 29 14:51:14 2016 -0700

--
 .../hadoop/hbase/regionserver/HRegion.java  |  19 ++--
 .../java/org/apache/hadoop/hbase/wal/WAL.java   |   2 +-
 .../hbase/regionserver/wal/TestFSHLog.java  | 101 ++-
 3 files changed, 110 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bf5a7aba/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index bfb9171..d06ed08 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2294,6 +2294,15 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 
 long trxId = 0;
 MultiVersionConcurrencyControl.WriteEntry writeEntry = mvcc.begin();
+// wait for all in-progress transactions to commit to WAL before
+// we can start the flush. This prevents
+// uncommitted transactions from being written into HFiles.
+// We have to block before we start the flush, otherwise keys that
+// were removed via a rollbackMemstore could be written to Hfiles.
+mvcc.completeAndWait(writeEntry);
+// set writeEntry to null to prevent mvcc.complete from being called again 
inside finally
+// block
+writeEntry = null;
 try {
   try {
 if (wal != null) {
@@ -2372,16 +2381,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   throw ioe;
 }
   }
-
-  // wait for all in-progress transactions to commit to WAL before
-  // we can start the flush. This prevents
-  // uncommitted transactions from being written into HFiles.
-  // We have to block before we start the flush, otherwise keys that
-  // were removed via a rollbackMemstore could be written to Hfiles.
-  mvcc.completeAndWait(writeEntry);
-  // set writeEntry to null to prevent mvcc.complete from being called 
again inside finally
-  // block
-  writeEntry = null;
 } finally {
   if (writeEntry != null) {
 // In case of failure just mark current writeEntry as complete.

http://git-wip-us.apache.org/repos/asf/hbase/blob/bf5a7aba/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 76b19f0..e43d44f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -51,7 +51,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL {
+public interface WAL extends AutoCloseable {
 
   /**
* Registers WALActionsListener

http://git-wip-us.apache.org/repos/asf/hbase/blob/bf5a7aba/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index 6ece700..760cdc1 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -30,6 +30,10 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.lang.mutable.MutableBoolean;

[3/3] hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking

[enis]

HBASE-16721 Concurrency issue in WAL unflushed seqId tracking


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cf374af1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cf374af1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cf374af1

Branch: refs/heads/branch-1.2
Commit: cf374af102f139a6176d05b97201bfa8d9f687be
Parents: 42dff8a
Author: Enis Soztutar 
Authored: Thu Sep 29 13:50:58 2016 -0700
Committer: Enis Soztutar 
Committed: Thu Sep 29 14:55:45 2016 -0700

--
 .../hadoop/hbase/regionserver/HRegion.java  |  19 ++--
 .../java/org/apache/hadoop/hbase/wal/WAL.java   |   2 +-
 .../hbase/regionserver/wal/TestFSHLog.java  | 100 ++-
 3 files changed, 109 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cf374af1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 22c66e3..f93b5a1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2216,6 +2216,15 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 
 long trxId = 0;
 MultiVersionConcurrencyControl.WriteEntry writeEntry = mvcc.begin();
+// wait for all in-progress transactions to commit to WAL before
+// we can start the flush. This prevents
+// uncommitted transactions from being written into HFiles.
+// We have to block before we start the flush, otherwise keys that
+// were removed via a rollbackMemstore could be written to Hfiles.
+mvcc.completeAndWait(writeEntry);
+// set writeEntry to null to prevent mvcc.complete from being called again 
inside finally
+// block
+writeEntry = null;
 try {
   try {
 if (wal != null) {
@@ -2294,16 +2303,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   throw ioe;
 }
   }
-
-  // wait for all in-progress transactions to commit to WAL before
-  // we can start the flush. This prevents
-  // uncommitted transactions from being written into HFiles.
-  // We have to block before we start the flush, otherwise keys that
-  // were removed via a rollbackMemstore could be written to Hfiles.
-  mvcc.completeAndWait(writeEntry);
-  // set writeEntry to null to prevent mvcc.complete from being called 
again inside finally
-  // block
-  writeEntry = null;
 } finally {
   if (writeEntry != null) {
 // In case of failure just mark current writeEntry as complete.

http://git-wip-us.apache.org/repos/asf/hbase/blob/cf374af1/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index d2b336e..041a5b9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -47,7 +47,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL {
+public interface WAL extends AutoCloseable {
 
   /**
* Registers WALActionsListener

http://git-wip-us.apache.org/repos/asf/hbase/blob/cf374af1/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index 6ece700..e09b621 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -30,6 +30,10 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.lang.mutable.MutableBoolean;
 import org.apache.commons.logging.Log;
@@ -55,6 +59,7 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import 

hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking

[enis]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 88512be52 -> 06c3dec2d


HBASE-16721 Concurrency issue in WAL unflushed seqId tracking

Conflicts:

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/06c3dec2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/06c3dec2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/06c3dec2

Branch: refs/heads/branch-1.1
Commit: 06c3dec2da32dcb588f0eb31e5db87796668bd39
Parents: 88512be
Author: Enis Soztutar 
Authored: Thu Sep 29 13:50:58 2016 -0700
Committer: Enis Soztutar 
Committed: Thu Sep 29 14:42:25 2016 -0700

--
 .../hadoop/hbase/regionserver/HRegion.java  |  19 ++--
 .../java/org/apache/hadoop/hbase/wal/WAL.java   |   2 +-
 .../hbase/regionserver/wal/TestFSHLog.java  | 101 ++-
 3 files changed, 110 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/06c3dec2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f033177..cc89b84 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2199,6 +2199,15 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 try {
   try {
 writeEntry = mvcc.beginMemstoreInsert();
+// wait for all in-progress transactions to commit to WAL before   

   
+// we can start the flush. This prevents
+// uncommitted transactions from being written into HFiles.
+// We have to block before we start the flush, otherwise keys that
+// were removed via a rollbackMemstore could be written to Hfiles.
+mvcc.waitForPreviousTransactionsComplete(writeEntry);
+// set w to null to prevent mvcc.advanceMemstore from being called 
again inside finally block
+writeEntry = null;
+
 if (wal != null) {
   Long earliestUnflushedSequenceIdForTheRegion =
   wal.startCacheFlush(encodedRegionName, flushedFamilyNames);
@@ -2275,16 +2284,6 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   throw ioe;
 }
   }
-
-  // wait for all in-progress transactions to commit to WAL before
-  // we can start the flush. This prevents
-  // uncommitted transactions from being written into HFiles.
-  // We have to block before we start the flush, otherwise keys that
-  // were removed via a rollbackMemstore could be written to Hfiles.
-  writeEntry.setWriteNumber(flushOpSeqId);
-  mvcc.waitForPreviousTransactionsComplete(writeEntry);
-  // set w to null to prevent mvcc.advanceMemstore from being called again 
inside finally block
-  writeEntry = null;
 } finally {
   if (writeEntry != null) {
 // in case of failure just mark current writeEntry as complete

http://git-wip-us.apache.org/repos/asf/hbase/blob/06c3dec2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 473bba9..20d0834 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -50,7 +50,7 @@ import com.google.common.annotations.VisibleForTesting;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL {
+public interface WAL extends AutoCloseable {
 
   /**
* Registers WALActionsListener

http://git-wip-us.apache.org/repos/asf/hbase/blob/06c3dec2/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index d14107a..1689778 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ 

hbase git commit: HBASE-16721 Concurrency issue in WAL unflushed seqId tracking

[enis]

Repository: hbase
Updated Branches:
  refs/heads/master 76396714e -> bf3c928b7


HBASE-16721 Concurrency issue in WAL unflushed seqId tracking


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bf3c928b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bf3c928b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bf3c928b

Branch: refs/heads/master
Commit: bf3c928b7499797735f71974992b68c9d876b97c
Parents: 7639671
Author: Enis Soztutar 
Authored: Thu Sep 29 12:56:22 2016 -0700
Committer: Enis Soztutar 
Committed: Thu Sep 29 12:56:22 2016 -0700

--
 .../java/org/apache/hadoop/hbase/wal/WAL.java   |   2 +-
 .../regionserver/wal/AbstractTestFSWAL.java |   2 +-
 .../hbase/regionserver/wal/TestFSHLog.java  | 105 +++
 3 files changed, 107 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bf3c928b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
--
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
index 79321b3..20ae602 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.hbase.util.Bytes;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public interface WAL {
+public interface WAL extends AutoCloseable {
 
   /**
* Registers WALActionsListener

http://git-wip-us.apache.org/repos/asf/hbase/blob/bf3c928b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
index 9eaeda4..19759d1 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestFSWAL.java
@@ -72,7 +72,7 @@ import org.junit.rules.TestName;
 
 public abstract class AbstractTestFSWAL {
 
-  private static final Log LOG = LogFactory.getLog(AbstractTestFSWAL.class);
+  protected static final Log LOG = LogFactory.getLog(AbstractTestFSWAL.class);
 
   protected static Configuration CONF;
   protected static FileSystem FS;

http://git-wip-us.apache.org/repos/asf/hbase/blob/bf3c928b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index bf56afe..640e851 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -23,6 +23,10 @@ import java.lang.reflect.Field;
 import java.util.List;
 import java.util.NavigableMap;
 import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -32,14 +36,21 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.wal.WALKey;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import static org.junit.Assert.assertEquals;
+
 /**
  * Provides FSHLog test cases.
  */
@@ -101,4 +112,98 @@ public class TestFSHLog extends AbstractTestFSWAL {
   log.close();
 }
   }
+
+  /**
+   * Test case for https://issues.apache.org/jira/browse/HBASE-16721
+   */
+  @Test (timeout = 3)
+  public void testUnflushedSeqIdTracking() throws IOException,