hbase git commit: HBASE-21095 The timeout retry logic for several procedures are broken after master restarts

2018-08-26 Thread zhangduo
Repository: hbase
Updated Branches:
  refs/heads/branch-2 f533f01a3 -> 404d4d56c


HBASE-21095 The timeout retry logic for several procedures are broken after 
master restarts


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/404d4d56
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/404d4d56
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/404d4d56

Branch: refs/heads/branch-2
Commit: 404d4d56cc681041777831a524f3c9be1cc8fcbe
Parents: f533f01
Author: zhangduo 
Authored: Thu Aug 23 22:16:13 2018 +0800
Committer: zhangduo 
Committed: Sun Aug 26 18:17:20 2018 +0800

--
 .../master/assignment/AssignmentManager.java| 67 +++-
 .../assignment/TransitRegionStateProcedure.java | 12 ++--
 .../master/procedure/ServerCrashProcedure.java  | 15 +++--
 .../assignment/TestCloseRegionWhileRSCrash.java | 11 +++-
 4 files changed, 75 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/404d4d56/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 9b020c8..a91f8e4 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -1414,12 +1414,25 @@ public class AssignmentManager implements 
ServerListener {
   //  Region Status update
   //  Should only be called in TransitRegionStateProcedure
   // 

+  private void transitStateAndUpdate(RegionStateNode regionNode, 
RegionState.State newState,
+  RegionState.State... expectedStates) throws IOException {
+RegionState.State state = regionNode.getState();
+regionNode.transitionState(newState, expectedStates);
+boolean succ = false;
+try {
+  regionStateStore.updateRegionLocation(regionNode);
+  succ = true;
+} finally {
+  if (!succ) {
+// revert
+regionNode.setState(state);
+  }
+}
+  }
 
   // should be called within the synchronized block of RegionStateNode
   void regionOpening(RegionStateNode regionNode) throws IOException {
-regionNode.transitionState(State.OPENING, 
RegionStates.STATES_EXPECTED_ON_OPEN);
-regionStateStore.updateRegionLocation(regionNode);
-
+transitStateAndUpdate(regionNode, State.OPENING, 
RegionStates.STATES_EXPECTED_ON_OPEN);
 regionStates.addRegionToServer(regionNode);
 // update the operation count metrics
 metrics.incrementOperationCounter();
@@ -1429,23 +1442,33 @@ public class AssignmentManager implements 
ServerListener {
   // The parameter 'giveUp' means whether we will try to open the region 
again, if it is true, then
   // we will persist the FAILED_OPEN state into hbase:meta.
   void regionFailedOpen(RegionStateNode regionNode, boolean giveUp) throws 
IOException {
-if (regionNode.getRegionLocation() != null) {
-  regionStates.removeRegionFromServer(regionNode.getRegionLocation(), 
regionNode);
-}
+RegionState.State state = regionNode.getState();
+ServerName regionLocation = regionNode.getRegionLocation();
 if (giveUp) {
   regionNode.setState(State.FAILED_OPEN);
   regionNode.setRegionLocation(null);
-  regionStateStore.updateRegionLocation(regionNode);
+  boolean succ = false;
+  try {
+regionStateStore.updateRegionLocation(regionNode);
+succ = true;
+  } finally {
+if (!succ) {
+  // revert
+  regionNode.setState(state);
+  regionNode.setRegionLocation(regionLocation);
+}
+  }
+}
+if (regionLocation != null) {
+  regionStates.removeRegionFromServer(regionLocation, regionNode);
 }
   }
 
   // should be called within the synchronized block of RegionStateNode
   void regionOpened(RegionStateNode regionNode) throws IOException {
-regionNode.transitionState(State.OPEN, 
RegionStates.STATES_EXPECTED_ON_OPEN);
 // TODO: OPENING Updates hbase:meta too... we need to do both here and 
there?
 // That is a lot of hbase:meta writing.
-regionStateStore.updateRegionLocation(regionNode);
-
+transitStateAndUpdate(regionNode, State.OPEN, 
RegionStates.STATES_EXPECTED_ON_OPEN);
 RegionInfo hri = regionNode.getRegionInfo();
 if (isMetaRegion(hri)) {
   // Usually we'd set a table ENABLED at this stage but hbase:meta is 
ALWAYs enabled, it
@@ -1460,7 +1483,7 @@ public class AssignmentManager implements 

hbase git commit: HBASE-21095 The timeout retry logic for several procedures are broken after master restarts

2018-08-24 Thread stack
Repository: hbase
Updated Branches:
  refs/heads/branch-2 2ea45e67d -> 09be1d0d5


HBASE-21095 The timeout retry logic for several procedures are broken after 
master restarts


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/09be1d0d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/09be1d0d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/09be1d0d

Branch: refs/heads/branch-2
Commit: 09be1d0d571f861a8f0c47560ebf120f34a44119
Parents: 2ea45e6
Author: Allan Yang 
Authored: Fri Aug 24 12:19:47 2018 -0700
Committer: Michael Stack 
Committed: Fri Aug 24 12:21:01 2018 -0700

--
 .../assignment/RegionTransitionProcedure.java | 11 ---
 .../assignment/TestUnexpectedStateException.java  | 18 ++
 2 files changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/09be1d0d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 0db8676..c10bf2d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -364,12 +364,9 @@ public abstract class RegionTransitionProcedure
   LOG.warn("Failed transition, suspend {}secs {}; {}; waiting on rectified 
condition fixed " +
   "by other Procedure or operator intervention", backoff / 1000, 
this,
   regionNode.toShortString(), e);
-  getRegionState(env).getProcedureEvent().suspend();
-  if (getRegionState(env).getProcedureEvent().suspendIfNotReady(this)) {
-setTimeout(Math.toIntExact(backoff));
-setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
-throw new ProcedureSuspendedException();
-  }
+  setTimeout(Math.toIntExact(backoff));
+  setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+  throw new ProcedureSuspendedException();
 }
 
 return new Procedure[] {this};
@@ -387,7 +384,7 @@ public abstract class RegionTransitionProcedure
   @Override
   protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
 setState(ProcedureProtos.ProcedureState.RUNNABLE);
-getRegionState(env).getProcedureEvent().wake(env.getProcedureScheduler());
+env.getProcedureScheduler().addFront(this);
 return false; // 'false' means that this procedure handled the timeout
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/09be1d0d/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
index 0f62f8e..16648c0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
@@ -24,17 +24,15 @@ import java.util.Iterator;
 import java.util.List;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
-import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
-import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
-import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -46,6 +44,11 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
+import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
+import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
+import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
+
 /**
  * Tests for 

hbase git commit: HBASE-21095 The timeout retry logic for several procedures are broken after master restarts

2018-08-24 Thread stack
Repository: hbase
Updated Branches:
  refs/heads/branch-2.1 66add5523 -> b82cd670c


HBASE-21095 The timeout retry logic for several procedures are broken after 
master restarts


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b82cd670
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b82cd670
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b82cd670

Branch: refs/heads/branch-2.1
Commit: b82cd670c3f473fde937c5b8445acc83ca76c5c6
Parents: 66add55
Author: Allan Yang 
Authored: Fri Aug 24 12:19:47 2018 -0700
Committer: Michael Stack 
Committed: Fri Aug 24 12:20:43 2018 -0700

--
 .../assignment/RegionTransitionProcedure.java | 11 ---
 .../assignment/TestUnexpectedStateException.java  | 18 ++
 2 files changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b82cd670/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 0db8676..c10bf2d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -364,12 +364,9 @@ public abstract class RegionTransitionProcedure
   LOG.warn("Failed transition, suspend {}secs {}; {}; waiting on rectified 
condition fixed " +
   "by other Procedure or operator intervention", backoff / 1000, 
this,
   regionNode.toShortString(), e);
-  getRegionState(env).getProcedureEvent().suspend();
-  if (getRegionState(env).getProcedureEvent().suspendIfNotReady(this)) {
-setTimeout(Math.toIntExact(backoff));
-setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
-throw new ProcedureSuspendedException();
-  }
+  setTimeout(Math.toIntExact(backoff));
+  setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+  throw new ProcedureSuspendedException();
 }
 
 return new Procedure[] {this};
@@ -387,7 +384,7 @@ public abstract class RegionTransitionProcedure
   @Override
   protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
 setState(ProcedureProtos.ProcedureState.RUNNABLE);
-getRegionState(env).getProcedureEvent().wake(env.getProcedureScheduler());
+env.getProcedureScheduler().addFront(this);
 return false; // 'false' means that this procedure handled the timeout
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/b82cd670/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
index 0f62f8e..16648c0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
@@ -24,17 +24,15 @@ import java.util.Iterator;
 import java.util.List;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
-import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
-import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
-import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -46,6 +44,11 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
+import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
+import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
+import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
+
 /**
  * Tests 

hbase git commit: HBASE-21095 The timeout retry logic for several procedures are broken after master restarts

2018-08-24 Thread stack
Repository: hbase
Updated Branches:
  refs/heads/branch-2.0 ba452da4b -> dae6b2f3f


HBASE-21095 The timeout retry logic for several procedures are broken after 
master restarts


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dae6b2f3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dae6b2f3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dae6b2f3

Branch: refs/heads/branch-2.0
Commit: dae6b2f3fbeb0e2af21cac9a3a3255416fbc9bf2
Parents: ba452da
Author: Allan Yang 
Authored: Fri Aug 24 12:19:47 2018 -0700
Committer: Michael Stack 
Committed: Fri Aug 24 12:19:47 2018 -0700

--
 .../assignment/RegionTransitionProcedure.java | 11 ---
 .../assignment/TestUnexpectedStateException.java  | 18 ++
 2 files changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/dae6b2f3/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index de39f4c..c9d141a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -371,12 +371,9 @@ public abstract class RegionTransitionProcedure
   LOG.warn("Failed transition, suspend {}secs {}; {}; waiting on rectified 
condition fixed " +
   "by other Procedure or operator intervention", backoff / 1000, 
this,
   regionNode.toShortString(), e);
-  getRegionState(env).getProcedureEvent().suspend();
-  if (getRegionState(env).getProcedureEvent().suspendIfNotReady(this)) {
-setTimeout(Math.toIntExact(backoff));
-setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
-throw new ProcedureSuspendedException();
-  }
+  setTimeout(Math.toIntExact(backoff));
+  setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+  throw new ProcedureSuspendedException();
 }
 
 return new Procedure[] {this};
@@ -394,7 +391,7 @@ public abstract class RegionTransitionProcedure
   @Override
   protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
 setState(ProcedureProtos.ProcedureState.RUNNABLE);
-getRegionState(env).getProcedureEvent().wake(env.getProcedureScheduler());
+env.getProcedureScheduler().addFront(this);
 return false; // 'false' means that this procedure handled the timeout
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/dae6b2f3/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
index 0f62f8e..16648c0 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestUnexpectedStateException.java
@@ -24,17 +24,15 @@ import java.util.Iterator;
 import java.util.List;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
-import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
-import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
-import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -46,6 +44,11 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.com.google.gson.JsonArray;
+import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
+import org.apache.hbase.thirdparty.com.google.gson.JsonObject;
+import org.apache.hbase.thirdparty.com.google.gson.JsonParser;
+
 /**
  * Tests 

hbase git commit: HBASE-21095 The timeout retry logic for several procedures are broken after master restarts

2018-08-23 Thread zhangduo
Repository: hbase
Updated Branches:
  refs/heads/master 780670ede -> aac1a7014


HBASE-21095 The timeout retry logic for several procedures are broken after 
master restarts


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aac1a701
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aac1a701
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aac1a701

Branch: refs/heads/master
Commit: aac1a70147ec950c98db5f31d8906ce54547523a
Parents: 780670e
Author: zhangduo 
Authored: Thu Aug 23 22:16:13 2018 +0800
Committer: Duo Zhang 
Committed: Fri Aug 24 10:07:31 2018 +0800

--
 .../master/assignment/AssignmentManager.java| 67 +++-
 .../assignment/TransitRegionStateProcedure.java | 12 ++--
 .../master/procedure/ServerCrashProcedure.java  | 15 +++--
 .../assignment/TestCloseRegionWhileRSCrash.java | 11 +++-
 4 files changed, 75 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/aac1a701/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 9b020c8..a91f8e4 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -1414,12 +1414,25 @@ public class AssignmentManager implements 
ServerListener {
   //  Region Status update
   //  Should only be called in TransitRegionStateProcedure
   // 

+  private void transitStateAndUpdate(RegionStateNode regionNode, 
RegionState.State newState,
+  RegionState.State... expectedStates) throws IOException {
+RegionState.State state = regionNode.getState();
+regionNode.transitionState(newState, expectedStates);
+boolean succ = false;
+try {
+  regionStateStore.updateRegionLocation(regionNode);
+  succ = true;
+} finally {
+  if (!succ) {
+// revert
+regionNode.setState(state);
+  }
+}
+  }
 
   // should be called within the synchronized block of RegionStateNode
   void regionOpening(RegionStateNode regionNode) throws IOException {
-regionNode.transitionState(State.OPENING, 
RegionStates.STATES_EXPECTED_ON_OPEN);
-regionStateStore.updateRegionLocation(regionNode);
-
+transitStateAndUpdate(regionNode, State.OPENING, 
RegionStates.STATES_EXPECTED_ON_OPEN);
 regionStates.addRegionToServer(regionNode);
 // update the operation count metrics
 metrics.incrementOperationCounter();
@@ -1429,23 +1442,33 @@ public class AssignmentManager implements 
ServerListener {
   // The parameter 'giveUp' means whether we will try to open the region 
again, if it is true, then
   // we will persist the FAILED_OPEN state into hbase:meta.
   void regionFailedOpen(RegionStateNode regionNode, boolean giveUp) throws 
IOException {
-if (regionNode.getRegionLocation() != null) {
-  regionStates.removeRegionFromServer(regionNode.getRegionLocation(), 
regionNode);
-}
+RegionState.State state = regionNode.getState();
+ServerName regionLocation = regionNode.getRegionLocation();
 if (giveUp) {
   regionNode.setState(State.FAILED_OPEN);
   regionNode.setRegionLocation(null);
-  regionStateStore.updateRegionLocation(regionNode);
+  boolean succ = false;
+  try {
+regionStateStore.updateRegionLocation(regionNode);
+succ = true;
+  } finally {
+if (!succ) {
+  // revert
+  regionNode.setState(state);
+  regionNode.setRegionLocation(regionLocation);
+}
+  }
+}
+if (regionLocation != null) {
+  regionStates.removeRegionFromServer(regionLocation, regionNode);
 }
   }
 
   // should be called within the synchronized block of RegionStateNode
   void regionOpened(RegionStateNode regionNode) throws IOException {
-regionNode.transitionState(State.OPEN, 
RegionStates.STATES_EXPECTED_ON_OPEN);
 // TODO: OPENING Updates hbase:meta too... we need to do both here and 
there?
 // That is a lot of hbase:meta writing.
-regionStateStore.updateRegionLocation(regionNode);
-
+transitStateAndUpdate(regionNode, State.OPEN, 
RegionStates.STATES_EXPECTED_ON_OPEN);
 RegionInfo hri = regionNode.getRegionInfo();
 if (isMetaRegion(hri)) {
   // Usually we'd set a table ENABLED at this stage but hbase:meta is 
ALWAYs enabled, it
@@ -1460,7 +1483,7 @@ public class AssignmentManager implements