[37/51] [abbrv] hbase git commit: HBASE-21534 TestAssignmentManager is flakey

2018-12-07 Thread elserj 
HBASE-21534 TestAssignmentManager is flakey


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d525ec6a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d525ec6a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d525ec6a

Branch: refs/heads/HBASE-20952
Commit: d525ec6a1214f97bda560095f9775ca96d82f030
Parents: 5c209f2
Author: Duo Zhang 
Authored: Fri Nov 30 15:26:04 2018 +0800
Committer: zhangduo 
Committed: Sat Dec 1 20:41:11 2018 +0800

--
 .../master/assignment/TestAssignmentManagerBase.java  | 14 ++
 1 file changed, 14 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d525ec6a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
index 7ab37bc..7b5c550 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotEquals;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.UncheckedIOException;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.NavigableMap;
@@ -38,8 +39,10 @@ import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerMetricsBuilder;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.YouAreDeadException;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
@@ -271,7 +274,18 @@ public abstract class TestAssignmentManagerBase {
   }
 
   protected void doCrash(final ServerName serverName) {
+this.master.getServerManager().moveFromOnlineToDeadServers(serverName);
 this.am.submitServerCrash(serverName, false/* No WALs here */);
+// add a new server to avoid killing all the region servers which may hang 
the UTs
+int maxPort = 
this.master.getServerManager().getOnlineServersList().stream()
+  .mapToInt(ServerName::getPort).max().getAsInt();
+ServerName newSn = ServerName.valueOf("localhost", 100 + maxPort + 1, 1);
+try {
+  this.master.getServerManager().regionServerReport(newSn, 
ServerMetricsBuilder.of(newSn));
+} catch (YouAreDeadException e) {
+  // should not happen
+  throw new UncheckedIOException(e);
+}
   }
 
   protected void doRestart(final ServerName serverName) {

hbase git commit: HBASE-21534 TestAssignmentManager is flakey

2018-12-05 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/branch-2 7e8ce1c5c -> 96f8e0cbe


HBASE-21534 TestAssignmentManager is flakey


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/96f8e0cb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/96f8e0cb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/96f8e0cb

Branch: refs/heads/branch-2
Commit: 96f8e0cbe9b6d1d6a5f4773c7b210b026c65b300
Parents: 7e8ce1c
Author: Duo Zhang 
Authored: Fri Nov 30 15:26:04 2018 +0800
Committer: Duo Zhang 
Committed: Thu Dec 6 09:54:18 2018 +0800

--
 .../assignment/TestAssignmentManagerBase.java   | 25 +---
 1 file changed, 22 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/96f8e0cb/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
index 7ab37bc..f666ab8 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotEquals;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.UncheckedIOException;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.NavigableMap;
@@ -38,8 +39,10 @@ import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerMetricsBuilder;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.YouAreDeadException;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
@@ -64,6 +67,8 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import 
org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
@@ -110,9 +115,11 @@ public abstract class TestAssignmentManagerBase {
   protected long unassignSubmittedCount = 0;
   protected long unassignFailedCount = 0;
 
+  protected int newRsAdded;
+
   protected int getAssignMaxAttempts() {
 // Have many so we succeed eventually.
-return 100;
+return 1000;
   }
 
   protected void setupConfiguration(Configuration conf) throws Exception {
@@ -127,11 +134,13 @@ public abstract class TestAssignmentManagerBase {
   @Before
   public void setUp() throws Exception {
 util = new HBaseTestingUtility();
-this.executor = Executors.newSingleThreadScheduledExecutor();
+this.executor = Executors.newSingleThreadScheduledExecutor(new 
ThreadFactoryBuilder()
+  .setUncaughtExceptionHandler((t, e) -> LOG.warn("Uncaught: ", 
e)).build());
 setupConfiguration(util.getConfiguration());
 master = new MockMasterServices(util.getConfiguration(), 
this.regionsToRegionServers);
 rsDispatcher = new MockRSProcedureDispatcher(master);
 master.start(NSERVERS, rsDispatcher);
+newRsAdded = 0;
 am = master.getAssignmentManager();
 assignProcMetrics = 
am.getAssignmentManagerMetrics().getAssignProcMetrics();
 unassignProcMetrics = 
am.getAssignmentManagerMetrics().getUnassignProcMetrics();
@@ -186,7 +195,7 @@ public abstract class TestAssignmentManagerBase {
 
   protected byte[] waitOnFuture(final Future future) throws Exception {
 try {
-  return future.get(60, TimeUnit.SECONDS);
+  return future.get(3, TimeUnit.MINUTES);
 } catch (ExecutionException e) {
   LOG.info("ExecutionException", e);
   Exception ee = (Exception) e.getCause();
@@ -271,7 +280,17 @@ public abstract class TestAssignmentManagerBase {
   }
 
   protected void doCrash(final ServerName serverName) {
+this.master.getServerManager().moveFromOnlineToDeadServers(serverName);
 this.am.submitServerCrash(serverName, false/* No WALs here */);
+// add a new server to avoid killing all the region servers which may hang 
the UTs
+ServerName newSn = ServerName.valueOf("localhost", 1 + newRsAdded, 1);
+

[2/6] hbase git commit: HBASE-21534 TestAssignmentManager is flakey

2018-12-03 Thread zhangduo 
HBASE-21534 TestAssignmentManager is flakey


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d525ec6a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d525ec6a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d525ec6a

Branch: refs/heads/HBASE-21512
Commit: d525ec6a1214f97bda560095f9775ca96d82f030
Parents: 5c209f2
Author: Duo Zhang 
Authored: Fri Nov 30 15:26:04 2018 +0800
Committer: zhangduo 
Committed: Sat Dec 1 20:41:11 2018 +0800

--
 .../master/assignment/TestAssignmentManagerBase.java  | 14 ++
 1 file changed, 14 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d525ec6a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
index 7ab37bc..7b5c550 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotEquals;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.UncheckedIOException;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.NavigableMap;
@@ -38,8 +39,10 @@ import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerMetricsBuilder;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.YouAreDeadException;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
@@ -271,7 +274,18 @@ public abstract class TestAssignmentManagerBase {
   }
 
   protected void doCrash(final ServerName serverName) {
+this.master.getServerManager().moveFromOnlineToDeadServers(serverName);
 this.am.submitServerCrash(serverName, false/* No WALs here */);
+// add a new server to avoid killing all the region servers which may hang 
the UTs
+int maxPort = 
this.master.getServerManager().getOnlineServersList().stream()
+  .mapToInt(ServerName::getPort).max().getAsInt();
+ServerName newSn = ServerName.valueOf("localhost", 100 + maxPort + 1, 1);
+try {
+  this.master.getServerManager().regionServerReport(newSn, 
ServerMetricsBuilder.of(newSn));
+} catch (YouAreDeadException e) {
+  // should not happen
+  throw new UncheckedIOException(e);
+}
   }
 
   protected void doRestart(final ServerName serverName) {

hbase git commit: HBASE-21534 TestAssignmentManager is flakey

2018-12-01 Thread zhangduo 
Repository: hbase
Updated Branches:
  refs/heads/master 5c209f2ce -> d525ec6a1


HBASE-21534 TestAssignmentManager is flakey


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d525ec6a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d525ec6a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d525ec6a

Branch: refs/heads/master
Commit: d525ec6a1214f97bda560095f9775ca96d82f030
Parents: 5c209f2
Author: Duo Zhang 
Authored: Fri Nov 30 15:26:04 2018 +0800
Committer: zhangduo 
Committed: Sat Dec 1 20:41:11 2018 +0800

--
 .../master/assignment/TestAssignmentManagerBase.java  | 14 ++
 1 file changed, 14 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d525ec6a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
index 7ab37bc..7b5c550 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManagerBase.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotEquals;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.UncheckedIOException;
 import java.net.SocketTimeoutException;
 import java.util.Arrays;
 import java.util.NavigableMap;
@@ -38,8 +39,10 @@ import java.util.concurrent.TimeUnit;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerMetricsBuilder;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.YouAreDeadException;
 import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
 import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
@@ -271,7 +274,18 @@ public abstract class TestAssignmentManagerBase {
   }
 
   protected void doCrash(final ServerName serverName) {
+this.master.getServerManager().moveFromOnlineToDeadServers(serverName);
 this.am.submitServerCrash(serverName, false/* No WALs here */);
+// add a new server to avoid killing all the region servers which may hang 
the UTs
+int maxPort = 
this.master.getServerManager().getOnlineServersList().stream()
+  .mapToInt(ServerName::getPort).max().getAsInt();
+ServerName newSn = ServerName.valueOf("localhost", 100 + maxPort + 1, 1);
+try {
+  this.master.getServerManager().regionServerReport(newSn, 
ServerMetricsBuilder.of(newSn));
+} catch (YouAreDeadException e) {
+  // should not happen
+  throw new UncheckedIOException(e);
+}
   }
 
   protected void doRestart(final ServerName serverName) {