hbase git commit: HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks

[chia7712]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 2e7f55fe3 -> 670e9431d


HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/670e9431
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/670e9431
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/670e9431

Branch: refs/heads/branch-1.2
Commit: 670e9431d40d35df4802bc0445012271ee904efc
Parents: 2e7f55f
Author: Chia-Ping Tsai 
Authored: Fri Jul 28 14:07:53 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Fri Jul 28 14:07:53 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |  28 ++--
 .../hbase/client/TestFromClientSide3.java   | 139 ++-
 2 files changed, 152 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/670e9431/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 3abca47..719336e 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7003,28 +7003,28 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
 
 MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
-boolean locked;
+boolean locked = false;
 boolean walSyncSuccessful = false;
-List acquiredRowLocks;
+List acquiredRowLocks = null;
 long addedSize = 0;
 List mutations = new ArrayList();
 Collection rowsToLock = processor.getRowsToLock();
 long mvccNum = 0;
 WALKey walKey = null;
 try {
-  // 2. Acquire the row lock(s)
-  acquiredRowLocks = new ArrayList(rowsToLock.size());
-  for (byte[] row : rowsToLock) {
-// Attempt to lock all involved rows, throw if any lock times out
-// use a writer lock for mixed reads and writes
-acquiredRowLocks.add(getRowLock(row));
-  }
-  // 3. Region lock
-  lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : 
acquiredRowLocks.size());
-  locked = true;
-
-  long now = EnvironmentEdgeManager.currentTime();
   try {
+// 2. Acquire the row lock(s)
+acquiredRowLocks = new ArrayList(rowsToLock.size());
+for (byte[] row : rowsToLock) {
+  // Attempt to lock all involved rows, throw if any lock times out
+  // use a writer lock for mixed reads and writes
+  acquiredRowLocks.add(getRowLock(row));
+}
+// 3. Region lock
+lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : 
acquiredRowLocks.size());
+locked = true;
+
+long now = EnvironmentEdgeManager.currentTime();
 // 4. Let the processor scan the rows, generate mutations and add
 //waledits
 doProcessRowWithTimeout(

http://git-wip-us.apache.org/repos/asf/hbase/blob/670e9431/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index 09c7e86..08ccc42 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -28,16 +29,36 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
-
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import static junit.framework.Assert.assertFalse;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import 

hbase git commit: HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks

[chia7712]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 488ae4be6 -> f18f916f0


HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f18f916f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f18f916f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f18f916f

Branch: refs/heads/branch-1.3
Commit: f18f916f050cf4dc106543d3dc7c6d2f78077661
Parents: 488ae4b
Author: Chia-Ping Tsai 
Authored: Fri Jul 28 03:06:41 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Fri Jul 28 14:07:01 2017 +0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |  28 ++--
 .../hbase/client/TestFromClientSide3.java   | 139 ++-
 2 files changed, 152 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f18f916f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index c2dae91..42a2389 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7203,28 +7203,28 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
 
 MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
-boolean locked;
+boolean locked = false;
 boolean walSyncSuccessful = false;
-List acquiredRowLocks;
+List acquiredRowLocks = null;
 long addedSize = 0;
 List mutations = new ArrayList();
 Collection rowsToLock = processor.getRowsToLock();
 long mvccNum = 0;
 WALKey walKey = null;
 try {
-  // 2. Acquire the row lock(s)
-  acquiredRowLocks = new ArrayList(rowsToLock.size());
-  for (byte[] row : rowsToLock) {
-// Attempt to lock all involved rows, throw if any lock times out
-// use a writer lock for mixed reads and writes
-acquiredRowLocks.add(getRowLockInternal(row, false));
-  }
-  // 3. Region lock
-  lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : 
acquiredRowLocks.size());
-  locked = true;
-
-  long now = EnvironmentEdgeManager.currentTime();
   try {
+// 2. Acquire the row lock(s)
+acquiredRowLocks = new ArrayList(rowsToLock.size());
+for (byte[] row : rowsToLock) {
+  // Attempt to lock all involved rows, throw if any lock times out
+  // use a writer lock for mixed reads and writes
+  acquiredRowLocks.add(getRowLockInternal(row, false));
+}
+// 3. Region lock
+lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : 
acquiredRowLocks.size());
+locked = true;
+
+long now = EnvironmentEdgeManager.currentTime();
 // 4. Let the processor scan the rows, generate mutations and add
 //waledits
 doProcessRowWithTimeout(

http://git-wip-us.apache.org/repos/asf/hbase/blob/f18f916f/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index 09c7e86..08ccc42 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -28,16 +29,36 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
-
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import static junit.framework.Assert.assertFalse;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import 

hbase git commit: HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing Tsai)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 e30329e37 -> e8f056935


HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing 
Tsai)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8f05693
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8f05693
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8f05693

Branch: refs/heads/branch-1
Commit: e8f056935ba90bb46b960d3a6086a99730c95abd
Parents: e30329e
Author: tedyu 
Authored: Sun Nov 20 18:17:21 2016 -0800
Committer: tedyu 
Committed: Sun Nov 20 18:17:21 2016 -0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |  28 ++--
 .../hbase/client/TestFromClientSide3.java   | 139 ++-
 2 files changed, 152 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e8f05693/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4972fd3..8f8add8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7188,28 +7188,28 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 }
 
 MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
-boolean locked;
+boolean locked = false;
 boolean walSyncSuccessful = false;
-List acquiredRowLocks;
+List acquiredRowLocks = null;
 long addedSize = 0;
 List mutations = new ArrayList();
 Collection rowsToLock = processor.getRowsToLock();
 long mvccNum = 0;
 WALKey walKey = null;
 try {
-  // 2. Acquire the row lock(s)
-  acquiredRowLocks = new ArrayList(rowsToLock.size());
-  for (byte[] row : rowsToLock) {
-// Attempt to lock all involved rows, throw if any lock times out
-// use a writer lock for mixed reads and writes
-acquiredRowLocks.add(getRowLockInternal(row, false));
-  }
-  // 3. Region lock
-  lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : 
acquiredRowLocks.size());
-  locked = true;
-
-  long now = EnvironmentEdgeManager.currentTime();
   try {
+// 2. Acquire the row lock(s)
+acquiredRowLocks = new ArrayList(rowsToLock.size());
+for (byte[] row : rowsToLock) {
+  // Attempt to lock all involved rows, throw if any lock times out
+  // use a writer lock for mixed reads and writes
+  acquiredRowLocks.add(getRowLockInternal(row, false));
+}
+// 3. Region lock
+lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : 
acquiredRowLocks.size());
+locked = true;
+
+long now = EnvironmentEdgeManager.currentTime();
 // 4. Let the processor scan the rows, generate mutations and add
 //waledits
 doProcessRowWithTimeout(

http://git-wip-us.apache.org/repos/asf/hbase/blob/e8f05693/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index 09c7e86..08ccc42 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -28,16 +29,36 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
-
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import static junit.framework.Assert.assertFalse;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import 

hbase git commit: HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing Tsai)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/master ec9c9e201 -> bb645bcfd


HBASE-17131 Avoid livelock caused by HRegion#processRowsWithLocks (ChiaPing 
Tsai)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb645bcf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb645bcf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb645bcf

Branch: refs/heads/master
Commit: bb645bcfda74ad1f96b16e6e47543d44fbca5a98
Parents: ec9c9e2
Author: tedyu 
Authored: Sun Nov 20 07:09:02 2016 -0800
Committer: tedyu 
Committed: Sun Nov 20 07:09:02 2016 -0800

--
 .../hadoop/hbase/regionserver/HRegion.java  |  26 ++--
 .../hbase/client/TestFromClientSide3.java   | 124 +--
 2 files changed, 129 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bb645bcf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4d35b51..c372faa 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7044,8 +7044,8 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
   return;
 }
 
-boolean locked;
-List acquiredRowLocks;
+boolean locked = false;
+List acquiredRowLocks = null;
 List mutations = new ArrayList();
 Collection rowsToLock = processor.getRowsToLock();
 // This is assigned by mvcc either explicity in the below or in the guts 
of the WAL append
@@ -7053,19 +7053,19 @@ public class HRegion implements HeapSize, 
PropagatingConfigurationObserver, Regi
 WriteEntry writeEntry = null;
 MemstoreSize memstoreSize = new MemstoreSize();
 try {
-  // STEP 2. Acquire the row lock(s)
-  acquiredRowLocks = new ArrayList(rowsToLock.size());
-  for (byte[] row : rowsToLock) {
-// Attempt to lock all involved rows, throw if any lock times out
-// use a writer lock for mixed reads and writes
-acquiredRowLocks.add(getRowLockInternal(row, false));
-  }
-  // STEP 3. Region lock
-  lock(this.updatesLock.readLock(), acquiredRowLocks.size() == 0 ? 1 : 
acquiredRowLocks.size());
-  locked = true;
   boolean success = false;
-  long now = EnvironmentEdgeManager.currentTime();
   try {
+// STEP 2. Acquire the row lock(s)
+acquiredRowLocks = new ArrayList<>(rowsToLock.size());
+for (byte[] row : rowsToLock) {
+  // Attempt to lock all involved rows, throw if any lock times out
+  // use a writer lock for mixed reads and writes
+  acquiredRowLocks.add(getRowLockInternal(row, false));
+}
+// STEP 3. Region lock
+lock(this.updatesLock.readLock(), acquiredRowLocks.isEmpty() ? 1 : 
acquiredRowLocks.size());
+locked = true;
+long now = EnvironmentEdgeManager.currentTime();
 // STEP 4. Let the processor scan the rows, generate mutations and add 
waledits
 doProcessRowWithTimeout(processor, now, this, mutations, walEdit, 
timeout);
 if (!mutations.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/bb645bcf/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index 82fbe77..cbc97a2 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -23,13 +23,17 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -43,10 +47,17 @@ import org.apache.hadoop.hbase.regionserver.Region;