hbase git commit: HBASE-15650 Remove TimeRangeTracker as point of contention when many threads reading a StoreFile

2016-04-15 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 6f4b75861 -> 4b69faa19


HBASE-15650 Remove TimeRangeTracker as point of contention when many threads 
reading a StoreFile

Refactor so we use the immutable, unsynchronized TimeRange when doing
time-based checks at read time rather than use heavily synchronized
TimeRangeTracker; let TimeRangeTracker be for write-time only.

While in here, changed the Segment stuff so that when an immutable
segment, it uses TimeRange rather than TimeRangeTracker too.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
  Make allTime final.
  Add a includesTimeRange method copied from TimeRangeTracker.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
  Change name of a few methods so they match TimeRange methods that do
  same thing.
  (getTimeRangeTracker, getTimeRange, toTimeRange) add utility methods

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
  Change Reader to use TimeRange-based checks instead of
  TimeRangeTracker.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b69faa1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b69faa1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b69faa1

Branch: refs/heads/branch-1.3
Commit: 4b69faa1903303419dfcf027a2268524816c7a35
Parents: 6f4b758
Author: stack 
Authored: Fri Apr 15 10:56:56 2016 -0700
Committer: stack 
Committed: Fri Apr 15 20:12:49 2016 -0700

--
 .../org/apache/hadoop/hbase/io/TimeRange.java   | 33 +++--
 .../hbase/io/hfile/HFilePrettyPrinter.java  |  3 +-
 .../hbase/regionserver/DefaultMemStore.java |  6 +-
 .../hadoop/hbase/regionserver/StoreFile.java| 63 +++-
 .../hbase/regionserver/TimeRangeTracker.java| 78 
 .../hbase/mapreduce/TestHFileOutputFormat.java  |  7 +-
 .../hbase/mapreduce/TestHFileOutputFormat2.java |  8 +-
 .../hbase/regionserver/MockStoreFile.java   |  8 +-
 .../regionserver/TestTimeRangeTracker.java  | 10 +--
 9 files changed, 139 insertions(+), 77 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4b69faa1/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
index 672cc9d..2b70644 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
@@ -36,9 +36,13 @@ import org.apache.hadoop.hbase.util.Bytes;
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public class TimeRange {
-  private long minStamp = 0L;
-  private long maxStamp = Long.MAX_VALUE;
-  private boolean allTime = false;
+  static final long INITIAL_MIN_TIMESTAMP = 0L;
+  private static final long MIN_TIME = INITIAL_MIN_TIMESTAMP;
+  static final long INITIAL_MAX_TIMESTAMP = Long.MAX_VALUE;
+  static final long MAX_TIME = INITIAL_MAX_TIMESTAMP;
+  private long minStamp = MIN_TIME;
+  private long maxStamp = MAX_TIME;
+  private final boolean allTime;
 
   /**
* Default constructor.
@@ -58,6 +62,7 @@ public class TimeRange {
   @Deprecated
   public TimeRange(long minStamp) {
 this.minStamp = minStamp;
+this.allTime = this.minStamp == MIN_TIME;
   }
 
   /**
@@ -68,6 +73,7 @@ public class TimeRange {
   @Deprecated
   public TimeRange(byte [] minStamp) {
 this.minStamp = Bytes.toLong(minStamp);
+this.allTime = false;
   }
 
   /**
@@ -84,11 +90,12 @@ public class TimeRange {
   throw new IllegalArgumentException("Timestamp cannot be negative. 
minStamp:" + minStamp
 + ", maxStamp:" + maxStamp);
 }
-if(maxStamp < minStamp) {
+if (maxStamp < minStamp) {
   throw new IOException("maxStamp is smaller than minStamp");
 }
 this.minStamp = minStamp;
 this.maxStamp = maxStamp;
+this.allTime = this.minStamp == MIN_TIME && this.maxStamp == MAX_TIME;
   }
 
   /**
@@ -149,12 +156,28 @@ public class TimeRange {
* @return true if within TimeRange, false if not
*/
   public boolean withinTimeRange(long timestamp) {
-if(allTime) return true;
+if (this.allTime) {
+  return true;
+}
 // check if >= minStamp
 return (minStamp <= timestamp && timestamp < maxStamp);
   }
 
   /**
+   * Check if the range has any overlap with TimeRange
+   * @param tr TimeRange
+   * @return True if there is overlap, false otherwise
+   */
+  // This method came from TimeRangeTracker. We used to go there for this 
function but better
+  // to come here to the immutable, unsynchronized datastructure at

hbase git commit: HBASE-15650 Remove TimeRangeTracker as point of contention when many threads reading a StoreFile

2016-04-15 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 043ffd1db -> fd4992335


HBASE-15650 Remove TimeRangeTracker as point of contention when many threads 
reading a StoreFile

Refactor so we use the immutable, unsynchronized TimeRange when doing
time-based checks at read time rather than use heavily synchronized
TimeRangeTracker; let TimeRangeTracker be for write-time only.

While in here, changed the Segment stuff so that when an immutable
segment, it uses TimeRange rather than TimeRangeTracker too.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
  Make allTime final.
  Add a includesTimeRange method copied from TimeRangeTracker.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
  Change name of a few methods so they match TimeRange methods that do
  same thing.
  (getTimeRangeTracker, getTimeRange, toTimeRange) add utility methods

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
  Change Reader to use TimeRange-based checks instead of
  TimeRangeTracker.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fd499233
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fd499233
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fd499233

Branch: refs/heads/branch-1
Commit: fd49923350ab2a3ba1258f3e76ffec8bf2f60d55
Parents: 043ffd1
Author: stack 
Authored: Fri Apr 15 10:56:56 2016 -0700
Committer: stack 
Committed: Fri Apr 15 16:56:05 2016 -0700

--
 .../org/apache/hadoop/hbase/io/TimeRange.java   | 37 +++---
 .../hbase/io/hfile/HFilePrettyPrinter.java  |  3 +-
 .../hbase/regionserver/DefaultMemStore.java |  6 +-
 .../hadoop/hbase/regionserver/StoreFile.java| 63 +++-
 .../hbase/regionserver/TimeRangeTracker.java| 78 
 .../hbase/mapreduce/TestHFileOutputFormat.java  |  7 +-
 .../hbase/mapreduce/TestHFileOutputFormat2.java |  8 +-
 .../hbase/regionserver/MockStoreFile.java   |  8 +-
 .../regionserver/TestTimeRangeTracker.java  | 10 +--
 9 files changed, 137 insertions(+), 83 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fd499233/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
index 4ec062d..2b70644 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
@@ -36,11 +36,13 @@ import org.apache.hadoop.hbase.util.Bytes;
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public class TimeRange {
-  private static final long MIN_TIME = 0L;
-  private static final long MAX_TIME = Long.MAX_VALUE;
+  static final long INITIAL_MIN_TIMESTAMP = 0L;
+  private static final long MIN_TIME = INITIAL_MIN_TIMESTAMP;
+  static final long INITIAL_MAX_TIMESTAMP = Long.MAX_VALUE;
+  static final long MAX_TIME = INITIAL_MAX_TIMESTAMP;
   private long minStamp = MIN_TIME;
   private long maxStamp = MAX_TIME;
-  private boolean allTime = false;
+  private final boolean allTime;
 
   /**
* Default constructor.
@@ -60,9 +62,7 @@ public class TimeRange {
   @Deprecated
   public TimeRange(long minStamp) {
 this.minStamp = minStamp;
-if (this.minStamp == MIN_TIME){
-  this.allTime = true;
-}
+this.allTime = this.minStamp == MIN_TIME;
   }
 
   /**
@@ -73,6 +73,7 @@ public class TimeRange {
   @Deprecated
   public TimeRange(byte [] minStamp) {
 this.minStamp = Bytes.toLong(minStamp);
+this.allTime = false;
   }
 
   /**
@@ -89,14 +90,12 @@ public class TimeRange {
   throw new IllegalArgumentException("Timestamp cannot be negative. 
minStamp:" + minStamp
 + ", maxStamp:" + maxStamp);
 }
-if(maxStamp < minStamp) {
+if (maxStamp < minStamp) {
   throw new IOException("maxStamp is smaller than minStamp");
 }
 this.minStamp = minStamp;
 this.maxStamp = maxStamp;
-if (this.minStamp == MIN_TIME && this.maxStamp == MAX_TIME){
-  this.allTime = true;
-}
+this.allTime = this.minStamp == MIN_TIME && this.maxStamp == MAX_TIME;
   }
 
   /**
@@ -157,12 +156,28 @@ public class TimeRange {
* @return true if within TimeRange, false if not
*/
   public boolean withinTimeRange(long timestamp) {
-if(allTime) return true;
+if (this.allTime) {
+  return true;
+}
 // check if >= minStamp
 return (minStamp <= timestamp && timestamp < maxStamp);
   }
 
   /**
+   * Check if the range has any overlap with TimeRange
+   * @param tr TimeRange
+   * @return True if there is

hbase git commit: HBASE-15187 Integrate CSRF prevention filter to REST gateway

2016-04-15 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 6cf4bfe62 -> 043ffd1db


HBASE-15187 Integrate CSRF prevention filter to REST gateway


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/043ffd1d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/043ffd1d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/043ffd1d

Branch: refs/heads/branch-1
Commit: 043ffd1db1f971b7a61de667768b99d9efbd4427
Parents: 6cf4bfe
Author: tedyu 
Authored: Fri Apr 15 13:46:30 2016 -0700
Committer: tedyu 
Committed: Fri Apr 15 13:46:30 2016 -0700

--
 .../src/main/resources/hbase-default.xml|  25 ++
 .../apache/hadoop/hbase/rest/RESTServer.java|  86 +-
 .../apache/hadoop/hbase/rest/client/Client.java | 107 ++-
 .../rest/filter/RestCsrfPreventionFilter.java   | 287 +++
 .../hbase/rest/HBaseRESTTestingUtility.java |   2 +
 .../hadoop/hbase/rest/TestMultiRowResource.java |  49 +++-
 .../hadoop/hbase/rest/TestSchemaResource.java   |  66 -
 7 files changed, 587 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/043ffd1d/hbase-common/src/main/resources/hbase-default.xml
--
diff --git a/hbase-common/src/main/resources/hbase-default.xml 
b/hbase-common/src/main/resources/hbase-default.xml
index 94f1e6d..fae986e 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1355,6 +1355,31 @@ possible configurations would overwhelm and obscure the 
important.
   as the SimpleLoadBalancer).
 
   
+
+   hbase.rest.csrf.enabled
+   false
+  
+Set to true to enable protection against cross-site request forgery (CSRF)
+   
+
+
+  hbase.rest-csrf.browser-useragents-regex
+  ^Mozilla.*,^Opera.*
+  
+A comma-separated list of regular expressions used to match against an HTTP
+request's User-Agent header when protection against cross-site request
+forgery (CSRF) is enabled for REST server by setting
+hbase.rest.csrf.enabled to true.  If the incoming User-Agent matches
+any of these regular expressions, then the request is considered to be sent
+by a browser, and therefore CSRF prevention is enforced.  If the request's
+User-Agent does not match any of these regular expressions, then the 
request
+is considered to be sent by something other than a browser, such as 
scripted
+automation.  In this case, CSRF is not a potential attack vector, so
+the prevention is not enforced.  This helps achieve backwards-compatibility
+with existing automation that has not been updated to send the CSRF
+prevention header.
+  
+
   
 hbase.security.exec.permission.checks
 false

http://git-wip-us.apache.org/repos/asf/hbase/blob/043ffd1d/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index ad8c65d..9dac84a 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -19,9 +19,11 @@
 package org.apache.hadoop.hbase.rest;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
@@ -35,13 +37,17 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.http.HttpServer;
 import org.apache.hadoop.hbase.http.InfoServer;
 import org.apache.hadoop.hbase.rest.filter.AuthFilter;
+import org.apache.hadoop.hbase.rest.filter.RestCsrfPreventionFilter;
 import org.apache.hadoop.hbase.security.UserProvider;
 import org.apache.hadoop.hbase.util.DNS;
 import org.apache.hadoop.hbase.util.HttpServerUtil;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.util.StringUtils;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.nio.SelectChannelConnector;
@@ -66,6 +72,15 @@ import com.sun.jersey.spi.container.servlet.ServletContainer;
  */
 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
 public class RESTServer implements Constants {
+  static Log LOG

hbase git commit: HBASE-15650 Remove TimeRangeTracker as point of contention when many threads reading a StoreFile

2016-04-15 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/master 6930da781 -> d815211f5


HBASE-15650 Remove TimeRangeTracker as point of contention when many 
threads reading a StoreFile

Refactor so we use the immutable, unsynchronized TimeRange when doing
time-based checks at read time rather than use heavily synchronized
TimeRangeTracker; let TimeRangeTracker be for write-time only.

While in here, changed the Segment stuff so that when an immutable
segment, it uses TimeRange rather than TimeRangeTracker too.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
  Make allTime final.
  Add a includesTimeRange method copied from TimeRangeTracker.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
  Change name of a few methods so they match TimeRange methods that do
  same thing.
  (getTimeRangeTracker, getTimeRange, toTimeRange) add utility methods

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java
  Change ImmutableSegment so it uses a TimeRange rather than
  TimeRangeTracker.. it is read-only. Redo shouldSeek, getMinTimestamp,
  updateMetaInfo, and getTimeRangeTracker so we use TimeRange-based
  implementations instead of TimeRangeTracker implementations.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
  Implement shouldSeek, getMinTimestamp, updateMetaInfo, and
  getTimeRangeTracker using TimeRangeTracker.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
  Make methods that were using TimeRangeTracker abstract and instead
  have the implementations do these methods how they want either using
  TimeRangeTracker when a mutable segment or TimeRange when an immutable
  segment.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
  Change Reader to use TimeRange-based checks instead of
  TimeRangeTracker.

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d815211f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d815211f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d815211f

Branch: refs/heads/master
Commit: d815211f5a2583e4b03c627585709872cc711b7e
Parents: 6930da7
Author: stack 
Authored: Thu Apr 14 16:53:12 2016 -0700
Committer: stack 
Committed: Fri Apr 15 10:13:42 2016 -0700

--
 .../org/apache/hadoop/hbase/io/TimeRange.java   | 37 +++---
 .../hbase/io/hfile/HFilePrettyPrinter.java  | 27 ---
 .../hbase/mob/DefaultMobStoreFlusher.java   |  2 +-
 .../hbase/regionserver/AbstractMemStore.java|  3 +-
 .../hbase/regionserver/DefaultMemStore.java |  5 +-
 .../hbase/regionserver/ImmutableSegment.java| 27 ++-
 .../hbase/regionserver/MemStoreSnapshot.java|  3 +-
 .../hbase/regionserver/MutableSegment.java  | 28 +++-
 .../hadoop/hbase/regionserver/Segment.java  | 43 ---
 .../hadoop/hbase/regionserver/StoreFile.java| 22 ++
 .../hbase/regionserver/StoreFileReader.java | 14 ++--
 .../hbase/regionserver/StoreFileWriter.java | 26 ---
 .../hbase/regionserver/TimeRangeTracker.java| 76 
 .../regionserver/compactions/Compactor.java | 10 +--
 .../hbase/mapreduce/TestHFileOutputFormat2.java |  8 +--
 .../hbase/regionserver/MockStoreFile.java   |  8 +--
 .../regionserver/TestTimeRangeTracker.java  | 10 +--
 17 files changed, 213 insertions(+), 136 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d815211f/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
index 212ad45..d5b2509 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
@@ -36,11 +36,13 @@ import org.apache.hadoop.hbase.util.Bytes;
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public class TimeRange {
-  private static final long MIN_TIME = 0L;
-  private static final long MAX_TIME = Long.MAX_VALUE;
+  static final long INITIAL_MIN_TIMESTAMP = 0l;
+  private static final long MIN_TIME = INITIAL_MIN_TIMESTAMP;
+  static final long INITIAL_MAX_TIMESTAMP = Long.MAX_VALUE;
+  static final long MAX_TIME = INITIAL_MAX_TIMESTAMP;
   private long minStamp = MIN_TIME;
   private long maxStamp = MAX_TIME;
-  private boolean allTime = false;
+  private final boolean allTime;
 
   /**
*

hbase git commit: HBASE-15187 Integrate CSRF prevention filter to REST gateway

2016-04-15 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/master 3e42f8260 -> 6930da781


HBASE-15187 Integrate CSRF prevention filter to REST gateway


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6930da78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6930da78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6930da78

Branch: refs/heads/master
Commit: 6930da781b2142ed8a0e6fc67095453dea38a072
Parents: 3e42f82
Author: tedyu 
Authored: Fri Apr 15 09:38:31 2016 -0700
Committer: tedyu 
Committed: Fri Apr 15 09:38:31 2016 -0700

--
 .../src/main/resources/hbase-default.xml|  25 ++
 .../apache/hadoop/hbase/rest/RESTServer.java|  86 +-
 .../apache/hadoop/hbase/rest/client/Client.java | 107 ++-
 .../rest/filter/RestCsrfPreventionFilter.java   | 287 +++
 .../hbase/rest/HBaseRESTTestingUtility.java |   2 +
 .../hadoop/hbase/rest/TestMultiRowResource.java |  49 +++-
 .../hadoop/hbase/rest/TestSchemaResource.java   |  66 -
 7 files changed, 587 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6930da78/hbase-common/src/main/resources/hbase-default.xml
--
diff --git a/hbase-common/src/main/resources/hbase-default.xml 
b/hbase-common/src/main/resources/hbase-default.xml
index e50e89e..91c0501 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1467,6 +1467,31 @@ possible configurations would overwhelm and obscure the 
important.
   
http://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.html
 
   
+
+   hbase.rest.csrf.enabled
+   false
+  
+Set to true to enable protection against cross-site request forgery (CSRF)
+   
+
+
+  hbase.rest-csrf.browser-useragents-regex
+  ^Mozilla.*,^Opera.*
+  
+A comma-separated list of regular expressions used to match against an HTTP
+request's User-Agent header when protection against cross-site request
+forgery (CSRF) is enabled for REST server by setting
+hbase.rest.csrf.enabled to true.  If the incoming User-Agent matches
+any of these regular expressions, then the request is considered to be sent
+by a browser, and therefore CSRF prevention is enforced.  If the request's
+User-Agent does not match any of these regular expressions, then the 
request
+is considered to be sent by something other than a browser, such as 
scripted
+automation.  In this case, CSRF is not a potential attack vector, so
+the prevention is not enforced.  This helps achieve backwards-compatibility
+with existing automation that has not been updated to send the CSRF
+prevention header.
+  
+
   
 hbase.security.exec.permission.checks
 false

http://git-wip-us.apache.org/repos/asf/hbase/blob/6930da78/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index 0460b0a..b26de54 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -19,9 +19,11 @@
 package org.apache.hadoop.hbase.rest;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
@@ -35,13 +37,17 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.http.HttpServer;
 import org.apache.hadoop.hbase.http.InfoServer;
 import org.apache.hadoop.hbase.rest.filter.AuthFilter;
+import org.apache.hadoop.hbase.rest.filter.RestCsrfPreventionFilter;
 import org.apache.hadoop.hbase.security.UserProvider;
 import org.apache.hadoop.hbase.util.DNS;
 import org.apache.hadoop.hbase.util.HttpServerUtil;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Strings;
 import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.hadoop.util.StringUtils;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.nio.SelectChannelConnector;
@@ -66,6 +72,15 @@ import com.sun.jersey.spi.container.servlet.ServletContainer;
  */

[5/6] hbase git commit: HBASE-15622 Superusers does not consider the keytab credentials

2016-04-15 Thread mbertozzi 
HBASE-15622 Superusers does not consider the keytab credentials


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1bca4820
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1bca4820
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1bca4820

Branch: refs/heads/branch-1.1
Commit: 1bca4820f90d97de324641478658ec1b7902e0d5
Parents: b6a5a51
Author: Matteo Bertozzi 
Authored: Fri Apr 15 08:28:39 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Apr 15 08:36:35 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1bca4820/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index da36daa..bea4283 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -498,7 +498,6 @@ public class HRegionServer extends HasThread implements
 this.conf = conf;
 checkCodecs(this.conf);
 this.userProvider = UserProvider.instantiate(conf);
-Superusers.initialize(conf);
 FSUtils.setupShortCircuitRead(this.conf);
 // Disable usage of meta replicas in the regionserver
 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
@@ -547,6 +546,9 @@ public class HRegionServer extends HasThread implements
   HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName);
 // login the server principal (if using secure Hadoop)
 login(userProvider, hostName);
+// init superusers and add the server principal (if using security)
+// or process owner as default super user.
+Superusers.initialize(conf);
 
 regionServerAccounting = new RegionServerAccounting();
 uncaughtExceptionHandler = new UncaughtExceptionHandler() {

[1/6] hbase git commit: HBASE-15622 Superusers does not consider the keytab credentials

2016-04-15 Thread mbertozzi 
Repository: hbase
Updated Branches:
  refs/heads/0.98 da785f38d -> a3846b132
  refs/heads/branch-1 96e9c466d -> 6cf4bfe62
  refs/heads/branch-1.1 b6a5a5164 -> 1bca4820f
  refs/heads/branch-1.2 cdf56ec93 -> c86d17d41
  refs/heads/branch-1.3 438739dc1 -> 6f4b75861
  refs/heads/master a9f1c4a87 -> 3e42f8260


HBASE-15622 Superusers does not consider the keytab credentials


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3e42f826
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3e42f826
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3e42f826

Branch: refs/heads/master
Commit: 3e42f8260009f01509bc9d8ba7e65fc5c75073e3
Parents: a9f1c4a
Author: Matteo Bertozzi 
Authored: Fri Apr 15 08:28:39 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Apr 15 08:28:39 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3e42f826/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1476190..b583e0e 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -517,7 +517,6 @@ public class HRegionServer extends HasThread implements
 HFile.checkHFileVersion(this.conf);
 checkCodecs(this.conf);
 this.userProvider = UserProvider.instantiate(conf);
-Superusers.initialize(conf);
 FSUtils.setupShortCircuitRead(this.conf);
 // Disable usage of meta replicas in the regionserver
 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
@@ -566,6 +565,9 @@ public class HRegionServer extends HasThread implements
   HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName);
 // login the server principal (if using secure Hadoop)
 login(userProvider, hostName);
+// init superusers and add the server principal (if using security)
+// or process owner as default super user.
+Superusers.initialize(conf);
 
 regionServerAccounting = new RegionServerAccounting();
 cacheConfig = new CacheConfig(conf);

[2/6] hbase git commit: HBASE-15622 Superusers does not consider the keytab credentials

2016-04-15 Thread mbertozzi 
HBASE-15622 Superusers does not consider the keytab credentials


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6cf4bfe6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6cf4bfe6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6cf4bfe6

Branch: refs/heads/branch-1
Commit: 6cf4bfe62c4cd0eee27005d96f7775fe34368f4a
Parents: 96e9c46
Author: Matteo Bertozzi 
Authored: Fri Apr 15 08:28:39 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Apr 15 08:31:46 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6cf4bfe6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9b89cd0..ff4afbc 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -511,7 +511,6 @@ public class HRegionServer extends HasThread implements
 this.conf = conf;
 checkCodecs(this.conf);
 this.userProvider = UserProvider.instantiate(conf);
-Superusers.initialize(conf);
 FSUtils.setupShortCircuitRead(this.conf);
 // Disable usage of meta replicas in the regionserver
 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
@@ -560,6 +559,9 @@ public class HRegionServer extends HasThread implements
   HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName);
 // login the server principal (if using secure Hadoop)
 login(userProvider, hostName);
+// init superusers and add the server principal (if using security)
+// or process owner as default super user.
+Superusers.initialize(conf);
 
 regionServerAccounting = new RegionServerAccounting();
 uncaughtExceptionHandler = new UncaughtExceptionHandler() {

[6/6] hbase git commit: HBASE-15622 Superusers does not consider the keytab credentials

2016-04-15 Thread mbertozzi 
HBASE-15622 Superusers does not consider the keytab credentials


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a3846b13
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a3846b13
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a3846b13

Branch: refs/heads/0.98
Commit: a3846b1329f5554225351d6142c33650c1c7d9db
Parents: da785f3
Author: Matteo Bertozzi 
Authored: Fri Apr 15 08:40:40 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Apr 15 08:40:40 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a3846b13/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 65e52b5..5061ce1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -565,8 +565,6 @@ public class HRegionServer implements 
ClientProtos.ClientService.BlockingInterfa
 checkCodecs(this.conf);
 this.userProvider = UserProvider.instantiate(conf);
 
-Superusers.initialize(conf);
-
 FSUtils.setupShortCircuitRead(this.conf);
 
 // Config'ed params
@@ -646,6 +644,8 @@ public class HRegionServer implements 
ClientProtos.ClientService.BlockingInterfa
 // login the server principal (if using secure Hadoop)
 userProvider.login("hbase.regionserver.keytab.file",
   "hbase.regionserver.kerberos.principal", this.isa.getHostName());
+Superusers.initialize(conf);
+
 regionServerAccounting = new RegionServerAccounting();
 cacheConfig = new CacheConfig(conf);
 uncaughtExceptionHandler = new UncaughtExceptionHandler() {

[3/6] hbase git commit: HBASE-15622 Superusers does not consider the keytab credentials

2016-04-15 Thread mbertozzi 
HBASE-15622 Superusers does not consider the keytab credentials


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6f4b7586
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6f4b7586
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6f4b7586

Branch: refs/heads/branch-1.3
Commit: 6f4b75861794f00a9093c9e4f0614108dd3f89b6
Parents: 438739d
Author: Matteo Bertozzi 
Authored: Fri Apr 15 08:28:39 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Apr 15 08:33:22 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6f4b7586/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9b89cd0..ff4afbc 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -511,7 +511,6 @@ public class HRegionServer extends HasThread implements
 this.conf = conf;
 checkCodecs(this.conf);
 this.userProvider = UserProvider.instantiate(conf);
-Superusers.initialize(conf);
 FSUtils.setupShortCircuitRead(this.conf);
 // Disable usage of meta replicas in the regionserver
 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
@@ -560,6 +559,9 @@ public class HRegionServer extends HasThread implements
   HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName);
 // login the server principal (if using secure Hadoop)
 login(userProvider, hostName);
+// init superusers and add the server principal (if using security)
+// or process owner as default super user.
+Superusers.initialize(conf);
 
 regionServerAccounting = new RegionServerAccounting();
 uncaughtExceptionHandler = new UncaughtExceptionHandler() {

[4/6] hbase git commit: HBASE-15622 Superusers does not consider the keytab credentials

2016-04-15 Thread mbertozzi 
HBASE-15622 Superusers does not consider the keytab credentials


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c86d17d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c86d17d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c86d17d4

Branch: refs/heads/branch-1.2
Commit: c86d17d416251a9bfad11aa98f4c9968e23e2b35
Parents: cdf56ec
Author: Matteo Bertozzi 
Authored: Fri Apr 15 08:28:39 2016 -0700
Committer: Matteo Bertozzi 
Committed: Fri Apr 15 08:35:28 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c86d17d4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 68c5ddf..816659a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -502,7 +502,6 @@ public class HRegionServer extends HasThread implements
 this.conf = conf;
 checkCodecs(this.conf);
 this.userProvider = UserProvider.instantiate(conf);
-Superusers.initialize(conf);
 FSUtils.setupShortCircuitRead(this.conf);
 // Disable usage of meta replicas in the regionserver
 this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);
@@ -551,6 +550,9 @@ public class HRegionServer extends HasThread implements
   HConstants.ZK_CLIENT_KERBEROS_PRINCIPAL, hostName);
 // login the server principal (if using secure Hadoop)
 login(userProvider, hostName);
+// init superusers and add the server principal (if using security)
+// or process owner as default super user.
+Superusers.initialize(conf);
 
 regionServerAccounting = new RegionServerAccounting();
 uncaughtExceptionHandler = new UncaughtExceptionHandler() {

[2/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
http://git-wip-us.apache.org/repos/asf/hbase/blob/438739dc/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index 588cc86..39619c4 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -28114,6 +28114,16 @@ public final class MasterProtos {
  * repeated .hbase.pb.MasterSwitchType switch_types = 3;
  */
 org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterSwitchType 
getSwitchTypes(int index);
+
+// optional bool skip_lock = 4;
+/**
+ * optional bool skip_lock = 4;
+ */
+boolean hasSkipLock();
+/**
+ * optional bool skip_lock = 4;
+ */
+boolean getSkipLock();
   }
   /**
* Protobuf type {@code hbase.pb.SetSplitOrMergeEnabledRequest}
@@ -28209,6 +28219,11 @@ public final class MasterProtos {
   input.popLimit(oldLimit);
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  skipLock_ = input.readBool();
+  break;
+}
   }
 }
   } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -28306,10 +28321,27 @@ public final class MasterProtos {
   return switchTypes_.get(index);
 }
 
+// optional bool skip_lock = 4;
+public static final int SKIP_LOCK_FIELD_NUMBER = 4;
+private boolean skipLock_;
+/**
+ * optional bool skip_lock = 4;
+ */
+public boolean hasSkipLock() {
+  return ((bitField0_ & 0x0004) == 0x0004);
+}
+/**
+ * optional bool skip_lock = 4;
+ */
+public boolean getSkipLock() {
+  return skipLock_;
+}
+
 private void initFields() {
   enabled_ = false;
   synchronous_ = false;
   switchTypes_ = java.util.Collections.emptyList();
+  skipLock_ = false;
 }
 private byte memoizedIsInitialized = -1;
 public final boolean isInitialized() {
@@ -28336,6 +28368,9 @@ public final class MasterProtos {
   for (int i = 0; i < switchTypes_.size(); i++) {
 output.writeEnum(3, switchTypes_.get(i).getNumber());
   }
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+output.writeBool(4, skipLock_);
+  }
   getUnknownFields().writeTo(output);
 }
 
@@ -28362,6 +28397,10 @@ public final class MasterProtos {
 size += dataSize;
 size += 1 * switchTypes_.size();
   }
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+size += com.google.protobuf.CodedOutputStream
+  .computeBoolSize(4, skipLock_);
+  }
   size += getUnknownFields().getSerializedSize();
   memoizedSerializedSize = size;
   return size;
@@ -28397,6 +28436,11 @@ public final class MasterProtos {
   }
   result = result && getSwitchTypesList()
   .equals(other.getSwitchTypesList());
+  result = result && (hasSkipLock() == other.hasSkipLock());
+  if (hasSkipLock()) {
+result = result && (getSkipLock()
+== other.getSkipLock());
+  }
   result = result &&
   getUnknownFields().equals(other.getUnknownFields());
   return result;
@@ -28422,6 +28466,10 @@ public final class MasterProtos {
 hash = (37 * hash) + SWITCH_TYPES_FIELD_NUMBER;
 hash = (53 * hash) + hashEnumList(getSwitchTypesList());
   }
+  if (hasSkipLock()) {
+hash = (37 * hash) + SKIP_LOCK_FIELD_NUMBER;
+hash = (53 * hash) + hashBoolean(getSkipLock());
+  }
   hash = (29 * hash) + getUnknownFields().hashCode();
   memoizedHashCode = hash;
   return hash;
@@ -28537,6 +28585,8 @@ public final class MasterProtos {
 bitField0_ = (bitField0_ & ~0x0002);
 switchTypes_ = java.util.Collections.emptyList();
 bitField0_ = (bitField0_ & ~0x0004);
+skipLock_ = false;
+bitField0_ = (bitField0_ & ~0x0008);
 return this;
   }
 
@@ -28578,6 +28628,10 @@ public final class MasterProtos {
   bitField0_ = (bitField0_ & ~0x0004);
 }
 result.switchTypes_ = switchTypes_;
+if (((from_bitField0_ & 0x0008) == 0x0008)) {
+  to_bitField0_ |= 0x0004;
+}
+result.skipLock_ = skipLock_;
 result.bitField0_ = to_bitField0_;
 onBuilt();
 return result;
@@ -28610,6 +28664,9 @@ public final class MasterProtos {
   }
   onChanged();
 }
+if (other.hasSkipLock()) {
+  setSkipLock(other.getSkipLock());
+}
 this.mergeUnknownFields(other.getUnknownFields());
 return

[3/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
HBASE-15406 Split / merge switch left disabled after early termination of hbck

Conflicts:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java

hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java

hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java

hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java

hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/438739dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/438739dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/438739dc

Branch: refs/heads/branch-1.3
Commit: 438739dc15ac17c7d0f8b85fa42aceaa33c5b6cc
Parents: 6d2dc2a
Author: chenheng 
Authored: Fri Apr 15 15:24:20 2016 +0800
Committer: chenheng 
Committed: Fri Apr 15 15:36:40 2016 +0800

--
 .../org/apache/hadoop/hbase/client/Admin.java   |   13 +
 .../hadoop/hbase/client/ConnectionManager.java  |7 +
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |   18 +-
 .../hadoop/hbase/protobuf/RequestConverter.java |   11 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java   |   10 +
 .../hbase/protobuf/generated/MasterProtos.java  | 1891 +-
 .../protobuf/generated/ZooKeeperProtos.java |  553 -
 hbase-protocol/src/main/protobuf/Master.proto   |   13 +
 .../src/main/protobuf/ZooKeeper.proto   |   10 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   24 +
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  160 +-
 .../hbase/zookeeper/SplitOrMergeTracker.java|   85 +
 .../hbase/client/TestSplitOrMergeStatus.java|   39 +-
 .../apache/hadoop/hbase/util/TestHBaseFsck.java |   59 +-
 hbase-shell/src/main/ruby/hbase/admin.rb|2 +-
 15 files changed, 2273 insertions(+), 622 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/438739dc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index d9361b3..8fd855c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1490,10 +1490,15 @@ public interface Admin extends Abortable, Closeable {
*
* @param enabled enabled or not
* @param synchronous If true, it waits until current split() call, if 
outstanding, to return.
+   * @param skipLock if false, we will do lock before change switch.
+   * with the lock, other requests to change the switch will 
be rejected!
+   * And when you set it to be false,
+   * you should call {@link 
#releaseSplitOrMergeLockAndRollback()} by yourself
* @param switchTypes switchType list {@link MasterSwitchType}
* @return Previous switch value array
*/
   boolean[] setSplitOrMergeEnabled(final boolean enabled, final boolean 
synchronous,
+   final boolean skipLock,
final MasterSwitchType... switchTypes) 
throws IOException;
 
   /**
@@ -1503,6 +1508,14 @@ public interface Admin extends Abortable, Closeable {
*/
   boolean isSplitOrMergeEnabled(final MasterSwitchType switchType) throws 
IOException;
 
+  /**
+   *  You should call this method after you call
+   *  {@link #setSplitOrMergeEnabled(boolean, boolean, boolean, 
MasterSwitchType...)}
+   *  with skipLock be false, this method will release the lock created by 
above method
+   *  and rollback the switch state to be original state before you change 
switch
+   * */
+  void releaseSplitOrMergeLockAndRollback() throws IOException;
+
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
   public enum MasterSwitchType {

http://git-wip-us.apache.org/repos/asf/hbase/blob/438739dc/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java

[1/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 6d2dc2a8b -> 438739dc1


http://git-wip-us.apache.org/repos/asf/hbase/blob/438739dc/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
index 09479c4..b0a844a 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
@@ -9725,6 +9725,540 @@ public final class ZooKeeperProtos {
 // @@protoc_insertion_point(class_scope:hbase.pb.SwitchState)
   }
 
+  public interface SplitAndMergeStateOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// optional bool split_enabled = 1;
+/**
+ * optional bool split_enabled = 1;
+ */
+boolean hasSplitEnabled();
+/**
+ * optional bool split_enabled = 1;
+ */
+boolean getSplitEnabled();
+
+// optional bool merge_enabled = 2;
+/**
+ * optional bool merge_enabled = 2;
+ */
+boolean hasMergeEnabled();
+/**
+ * optional bool merge_enabled = 2;
+ */
+boolean getMergeEnabled();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.SplitAndMergeState}
+   *
+   * 
+   **
+   * State for split and merge, used in hbck
+   * 
+   */
+  public static final class SplitAndMergeState extends
+  com.google.protobuf.GeneratedMessage
+  implements SplitAndMergeStateOrBuilder {
+// Use SplitAndMergeState.newBuilder() to construct.
+private SplitAndMergeState(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private SplitAndMergeState(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SplitAndMergeState defaultInstance;
+public static SplitAndMergeState getDefaultInstance() {
+  return defaultInstance;
+}
+
+public SplitAndMergeState getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private SplitAndMergeState(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 8: {
+  bitField0_ |= 0x0001;
+  splitEnabled_ = input.readBool();
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  mergeEnabled_ = input.readBool();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.internal_static_hbase_pb_SplitAndMergeState_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.internal_static_hbase_pb_SplitAndMergeState_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitAndMergeState.class,
 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitAndMergeState.Builder.class);
+}
+
+public static com.google.protobuf.Parser PARSER =
+new com.google.protobuf.AbstractParser()

[1/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 d37897535 -> 96e9c466d


http://git-wip-us.apache.org/repos/asf/hbase/blob/96e9c466/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
index 09479c4..b0a844a 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
@@ -9725,6 +9725,540 @@ public final class ZooKeeperProtos {
 // @@protoc_insertion_point(class_scope:hbase.pb.SwitchState)
   }
 
+  public interface SplitAndMergeStateOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// optional bool split_enabled = 1;
+/**
+ * optional bool split_enabled = 1;
+ */
+boolean hasSplitEnabled();
+/**
+ * optional bool split_enabled = 1;
+ */
+boolean getSplitEnabled();
+
+// optional bool merge_enabled = 2;
+/**
+ * optional bool merge_enabled = 2;
+ */
+boolean hasMergeEnabled();
+/**
+ * optional bool merge_enabled = 2;
+ */
+boolean getMergeEnabled();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.SplitAndMergeState}
+   *
+   * 
+   **
+   * State for split and merge, used in hbck
+   * 
+   */
+  public static final class SplitAndMergeState extends
+  com.google.protobuf.GeneratedMessage
+  implements SplitAndMergeStateOrBuilder {
+// Use SplitAndMergeState.newBuilder() to construct.
+private SplitAndMergeState(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private SplitAndMergeState(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SplitAndMergeState defaultInstance;
+public static SplitAndMergeState getDefaultInstance() {
+  return defaultInstance;
+}
+
+public SplitAndMergeState getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private SplitAndMergeState(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 8: {
+  bitField0_ |= 0x0001;
+  splitEnabled_ = input.readBool();
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  mergeEnabled_ = input.readBool();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.internal_static_hbase_pb_SplitAndMergeState_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.internal_static_hbase_pb_SplitAndMergeState_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitAndMergeState.class,
 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitAndMergeState.Builder.class);
+}
+
+public static com.google.protobuf.Parser PARSER =
+new com.google.protobuf.AbstractParser() {

[2/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
http://git-wip-us.apache.org/repos/asf/hbase/blob/96e9c466/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index 588cc86..39619c4 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -28114,6 +28114,16 @@ public final class MasterProtos {
  * repeated .hbase.pb.MasterSwitchType switch_types = 3;
  */
 org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterSwitchType 
getSwitchTypes(int index);
+
+// optional bool skip_lock = 4;
+/**
+ * optional bool skip_lock = 4;
+ */
+boolean hasSkipLock();
+/**
+ * optional bool skip_lock = 4;
+ */
+boolean getSkipLock();
   }
   /**
* Protobuf type {@code hbase.pb.SetSplitOrMergeEnabledRequest}
@@ -28209,6 +28219,11 @@ public final class MasterProtos {
   input.popLimit(oldLimit);
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  skipLock_ = input.readBool();
+  break;
+}
   }
 }
   } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -28306,10 +28321,27 @@ public final class MasterProtos {
   return switchTypes_.get(index);
 }
 
+// optional bool skip_lock = 4;
+public static final int SKIP_LOCK_FIELD_NUMBER = 4;
+private boolean skipLock_;
+/**
+ * optional bool skip_lock = 4;
+ */
+public boolean hasSkipLock() {
+  return ((bitField0_ & 0x0004) == 0x0004);
+}
+/**
+ * optional bool skip_lock = 4;
+ */
+public boolean getSkipLock() {
+  return skipLock_;
+}
+
 private void initFields() {
   enabled_ = false;
   synchronous_ = false;
   switchTypes_ = java.util.Collections.emptyList();
+  skipLock_ = false;
 }
 private byte memoizedIsInitialized = -1;
 public final boolean isInitialized() {
@@ -28336,6 +28368,9 @@ public final class MasterProtos {
   for (int i = 0; i < switchTypes_.size(); i++) {
 output.writeEnum(3, switchTypes_.get(i).getNumber());
   }
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+output.writeBool(4, skipLock_);
+  }
   getUnknownFields().writeTo(output);
 }
 
@@ -28362,6 +28397,10 @@ public final class MasterProtos {
 size += dataSize;
 size += 1 * switchTypes_.size();
   }
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+size += com.google.protobuf.CodedOutputStream
+  .computeBoolSize(4, skipLock_);
+  }
   size += getUnknownFields().getSerializedSize();
   memoizedSerializedSize = size;
   return size;
@@ -28397,6 +28436,11 @@ public final class MasterProtos {
   }
   result = result && getSwitchTypesList()
   .equals(other.getSwitchTypesList());
+  result = result && (hasSkipLock() == other.hasSkipLock());
+  if (hasSkipLock()) {
+result = result && (getSkipLock()
+== other.getSkipLock());
+  }
   result = result &&
   getUnknownFields().equals(other.getUnknownFields());
   return result;
@@ -28422,6 +28466,10 @@ public final class MasterProtos {
 hash = (37 * hash) + SWITCH_TYPES_FIELD_NUMBER;
 hash = (53 * hash) + hashEnumList(getSwitchTypesList());
   }
+  if (hasSkipLock()) {
+hash = (37 * hash) + SKIP_LOCK_FIELD_NUMBER;
+hash = (53 * hash) + hashBoolean(getSkipLock());
+  }
   hash = (29 * hash) + getUnknownFields().hashCode();
   memoizedHashCode = hash;
   return hash;
@@ -28537,6 +28585,8 @@ public final class MasterProtos {
 bitField0_ = (bitField0_ & ~0x0002);
 switchTypes_ = java.util.Collections.emptyList();
 bitField0_ = (bitField0_ & ~0x0004);
+skipLock_ = false;
+bitField0_ = (bitField0_ & ~0x0008);
 return this;
   }
 
@@ -28578,6 +28628,10 @@ public final class MasterProtos {
   bitField0_ = (bitField0_ & ~0x0004);
 }
 result.switchTypes_ = switchTypes_;
+if (((from_bitField0_ & 0x0008) == 0x0008)) {
+  to_bitField0_ |= 0x0004;
+}
+result.skipLock_ = skipLock_;
 result.bitField0_ = to_bitField0_;
 onBuilt();
 return result;
@@ -28610,6 +28664,9 @@ public final class MasterProtos {
   }
   onChanged();
 }
+if (other.hasSkipLock()) {
+  setSkipLock(other.getSkipLock());
+}
 this.mergeUnknownFields(other.getUnknownFields());
 return

[3/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
HBASE-15406 Split / merge switch left disabled after early termination of hbck

Conflicts:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java

hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java

hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java

hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java

hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/96e9c466
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/96e9c466
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/96e9c466

Branch: refs/heads/branch-1
Commit: 96e9c466d2d69cc1d0ea4d64d0ed2bada2ec3391
Parents: d378975
Author: chenheng 
Authored: Fri Apr 15 15:24:20 2016 +0800
Committer: chenheng 
Committed: Fri Apr 15 15:34:43 2016 +0800

--
 .../org/apache/hadoop/hbase/client/Admin.java   |   13 +
 .../hadoop/hbase/client/ConnectionManager.java  |7 +
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |   18 +-
 .../hadoop/hbase/protobuf/RequestConverter.java |   11 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java   |   10 +
 .../hbase/protobuf/generated/MasterProtos.java  | 1891 +-
 .../protobuf/generated/ZooKeeperProtos.java |  553 -
 hbase-protocol/src/main/protobuf/Master.proto   |   13 +
 .../src/main/protobuf/ZooKeeper.proto   |   10 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   24 +
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |  160 +-
 .../hbase/zookeeper/SplitOrMergeTracker.java|   85 +
 .../hbase/client/TestSplitOrMergeStatus.java|   39 +-
 .../apache/hadoop/hbase/util/TestHBaseFsck.java |   59 +-
 hbase-shell/src/main/ruby/hbase/admin.rb|2 +-
 15 files changed, 2273 insertions(+), 622 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/96e9c466/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index d9361b3..8fd855c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1490,10 +1490,15 @@ public interface Admin extends Abortable, Closeable {
*
* @param enabled enabled or not
* @param synchronous If true, it waits until current split() call, if 
outstanding, to return.
+   * @param skipLock if false, we will do lock before change switch.
+   * with the lock, other requests to change the switch will 
be rejected!
+   * And when you set it to be false,
+   * you should call {@link 
#releaseSplitOrMergeLockAndRollback()} by yourself
* @param switchTypes switchType list {@link MasterSwitchType}
* @return Previous switch value array
*/
   boolean[] setSplitOrMergeEnabled(final boolean enabled, final boolean 
synchronous,
+   final boolean skipLock,
final MasterSwitchType... switchTypes) 
throws IOException;
 
   /**
@@ -1503,6 +1508,14 @@ public interface Admin extends Abortable, Closeable {
*/
   boolean isSplitOrMergeEnabled(final MasterSwitchType switchType) throws 
IOException;
 
+  /**
+   *  You should call this method after you call
+   *  {@link #setSplitOrMergeEnabled(boolean, boolean, boolean, 
MasterSwitchType...)}
+   *  with skipLock be false, this method will release the lock created by 
above method
+   *  and rollback the switch state to be original state before you change 
switch
+   * */
+  void releaseSplitOrMergeLockAndRollback() throws IOException;
+
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
   public enum MasterSwitchType {

http://git-wip-us.apache.org/repos/asf/hbase/blob/96e9c466/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java

[3/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
HBASE-15406 Split / merge switch left disabled after early termination of hbck


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a9f1c4a8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a9f1c4a8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a9f1c4a8

Branch: refs/heads/master
Commit: a9f1c4a87153002629c291312bac4b758ad58204
Parents: 7efb9ed
Author: chenheng 
Authored: Thu Apr 7 14:25:18 2016 +0800
Committer: chenheng 
Committed: Fri Apr 15 14:09:52 2016 +0800

--
 .../org/apache/hadoop/hbase/client/Admin.java   |   13 +
 .../hbase/client/ConnectionImplementation.java  |7 +
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |   18 +-
 .../hadoop/hbase/protobuf/RequestConverter.java |   11 +-
 .../hbase/zookeeper/ZooKeeperWatcher.java   |   10 +
 .../hbase/protobuf/generated/MasterProtos.java  | 1903 +-
 .../protobuf/generated/ZooKeeperProtos.java |  552 -
 hbase-protocol/src/main/protobuf/Master.proto   |   13 +
 .../src/main/protobuf/ZooKeeper.proto   |   10 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   23 +
 .../org/apache/hadoop/hbase/util/HBaseFsck.java |   61 +-
 .../hbase/zookeeper/SplitOrMergeTracker.java|   85 +
 .../hbase/client/TestSplitOrMergeStatus.java|   39 +-
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   |   55 +
 hbase-shell/src/main/ruby/hbase/admin.rb|2 +-
 15 files changed, 2235 insertions(+), 567 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a9f1c4a8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 54d2cb9..97356a2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1716,10 +1716,15 @@ public interface Admin extends Abortable, Closeable {
*
* @param enabled enabled or not
* @param synchronous If true, it waits until current split() call, if 
outstanding, to return.
+   * @param skipLock if false, we will do lock before change switch.
+   * with the lock, other requests to change the switch will 
be rejected!
+   * And when you set it to be false,
+   * you should call {@link 
#releaseSplitOrMergeLockAndRollback()} by yourself
* @param switchTypes switchType list {@link MasterSwitchType}
* @return Previous switch value array
*/
   boolean[] setSplitOrMergeEnabled(final boolean enabled, final boolean 
synchronous,
+   final boolean skipLock,
final MasterSwitchType... switchTypes) 
throws IOException;
 
   /**
@@ -1730,6 +1735,14 @@ public interface Admin extends Abortable, Closeable {
   boolean isSplitOrMergeEnabled(final MasterSwitchType switchType) throws 
IOException;
 
   /**
+   *  You should call this method after you call
+   *  {@link #setSplitOrMergeEnabled(boolean, boolean, boolean, 
MasterSwitchType...)}
+   *  with skipLock be false, this method will release the lock created by 
above method
+   *  and rollback the switch state to be original state before you change 
switch
+   * */
+  void releaseSplitOrMergeLockAndRollback() throws IOException;
+
+  /**
* Currently, there are only two compact types:
* {@code NORMAL} means do store files compaction;
* {@code MOB} means do mob files compaction.

http://git-wip-us.apache.org/repos/asf/hbase/blob/a9f1c4a8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index e43a712..21e7e51 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -1751,6 +1751,13 @@ class ConnectionImplementation implements 
ClusterConnection, Closeable {
   }
 
   @Override
+  public MasterProtos.ReleaseSplitOrMergeLockAndRollbackResponse
+releaseSplitOrMergeLockAndRollback(RpcController controller,
+MasterProtos.ReleaseSplitOrMergeLockAndRollbackRequest request) throws 
ServiceException {
+return stub.releaseSplitOrMergeLockAndRollback(controller, request);
+  }
+
+  @Override
   public IsNormalizerEnabledResponse

[1/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/master 7efb9edec -> a9f1c4a87


http://git-wip-us.apache.org/repos/asf/hbase/blob/a9f1c4a8/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
index f64d0c1..a45c421 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java
@@ -9733,6 +9733,540 @@ public final class ZooKeeperProtos {
 // @@protoc_insertion_point(class_scope:hbase.pb.SwitchState)
   }
 
+  public interface SplitAndMergeStateOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// optional bool split_enabled = 1;
+/**
+ * optional bool split_enabled = 1;
+ */
+boolean hasSplitEnabled();
+/**
+ * optional bool split_enabled = 1;
+ */
+boolean getSplitEnabled();
+
+// optional bool merge_enabled = 2;
+/**
+ * optional bool merge_enabled = 2;
+ */
+boolean hasMergeEnabled();
+/**
+ * optional bool merge_enabled = 2;
+ */
+boolean getMergeEnabled();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.SplitAndMergeState}
+   *
+   * 
+   **
+   * State for split and merge, used in hbck
+   * 
+   */
+  public static final class SplitAndMergeState extends
+  com.google.protobuf.GeneratedMessage
+  implements SplitAndMergeStateOrBuilder {
+// Use SplitAndMergeState.newBuilder() to construct.
+private SplitAndMergeState(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private SplitAndMergeState(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SplitAndMergeState defaultInstance;
+public static SplitAndMergeState getDefaultInstance() {
+  return defaultInstance;
+}
+
+public SplitAndMergeState getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private SplitAndMergeState(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 8: {
+  bitField0_ |= 0x0001;
+  splitEnabled_ = input.readBool();
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  mergeEnabled_ = input.readBool();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.internal_static_hbase_pb_SplitAndMergeState_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.internal_static_hbase_pb_SplitAndMergeState_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitAndMergeState.class,
 
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitAndMergeState.Builder.class);
+}
+
+public static com.google.protobuf.Parser PARSER =
+new com.google.protobuf.AbstractParser() {
+

[2/3] hbase git commit: HBASE-15406 Split / merge switch left disabled after early termination of hbck

2016-04-15 Thread chenheng 
http://git-wip-us.apache.org/repos/asf/hbase/blob/a9f1c4a8/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index b91a36b..b4bd7af 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -28882,6 +28882,16 @@ public final class MasterProtos {
  * repeated .hbase.pb.MasterSwitchType switch_types = 3;
  */
 org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterSwitchType 
getSwitchTypes(int index);
+
+// optional bool skip_lock = 4;
+/**
+ * optional bool skip_lock = 4;
+ */
+boolean hasSkipLock();
+/**
+ * optional bool skip_lock = 4;
+ */
+boolean getSkipLock();
   }
   /**
* Protobuf type {@code hbase.pb.SetSplitOrMergeEnabledRequest}
@@ -28977,6 +28987,11 @@ public final class MasterProtos {
   input.popLimit(oldLimit);
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  skipLock_ = input.readBool();
+  break;
+}
   }
 }
   } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -29074,10 +29089,27 @@ public final class MasterProtos {
   return switchTypes_.get(index);
 }
 
+// optional bool skip_lock = 4;
+public static final int SKIP_LOCK_FIELD_NUMBER = 4;
+private boolean skipLock_;
+/**
+ * optional bool skip_lock = 4;
+ */
+public boolean hasSkipLock() {
+  return ((bitField0_ & 0x0004) == 0x0004);
+}
+/**
+ * optional bool skip_lock = 4;
+ */
+public boolean getSkipLock() {
+  return skipLock_;
+}
+
 private void initFields() {
   enabled_ = false;
   synchronous_ = false;
   switchTypes_ = java.util.Collections.emptyList();
+  skipLock_ = false;
 }
 private byte memoizedIsInitialized = -1;
 public final boolean isInitialized() {
@@ -29104,6 +29136,9 @@ public final class MasterProtos {
   for (int i = 0; i < switchTypes_.size(); i++) {
 output.writeEnum(3, switchTypes_.get(i).getNumber());
   }
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+output.writeBool(4, skipLock_);
+  }
   getUnknownFields().writeTo(output);
 }
 
@@ -29130,6 +29165,10 @@ public final class MasterProtos {
 size += dataSize;
 size += 1 * switchTypes_.size();
   }
+  if (((bitField0_ & 0x0004) == 0x0004)) {
+size += com.google.protobuf.CodedOutputStream
+  .computeBoolSize(4, skipLock_);
+  }
   size += getUnknownFields().getSerializedSize();
   memoizedSerializedSize = size;
   return size;
@@ -29165,6 +29204,11 @@ public final class MasterProtos {
   }
   result = result && getSwitchTypesList()
   .equals(other.getSwitchTypesList());
+  result = result && (hasSkipLock() == other.hasSkipLock());
+  if (hasSkipLock()) {
+result = result && (getSkipLock()
+== other.getSkipLock());
+  }
   result = result &&
   getUnknownFields().equals(other.getUnknownFields());
   return result;
@@ -29190,6 +29234,10 @@ public final class MasterProtos {
 hash = (37 * hash) + SWITCH_TYPES_FIELD_NUMBER;
 hash = (53 * hash) + hashEnumList(getSwitchTypesList());
   }
+  if (hasSkipLock()) {
+hash = (37 * hash) + SKIP_LOCK_FIELD_NUMBER;
+hash = (53 * hash) + hashBoolean(getSkipLock());
+  }
   hash = (29 * hash) + getUnknownFields().hashCode();
   memoizedHashCode = hash;
   return hash;
@@ -29305,6 +29353,8 @@ public final class MasterProtos {
 bitField0_ = (bitField0_ & ~0x0002);
 switchTypes_ = java.util.Collections.emptyList();
 bitField0_ = (bitField0_ & ~0x0004);
+skipLock_ = false;
+bitField0_ = (bitField0_ & ~0x0008);
 return this;
   }
 
@@ -29346,6 +29396,10 @@ public final class MasterProtos {
   bitField0_ = (bitField0_ & ~0x0004);
 }
 result.switchTypes_ = switchTypes_;
+if (((from_bitField0_ & 0x0008) == 0x0008)) {
+  to_bitField0_ |= 0x0004;
+}
+result.skipLock_ = skipLock_;
 result.bitField0_ = to_bitField0_;
 onBuilt();
 return result;
@@ -29378,6 +29432,9 @@ public final class MasterProtos {
   }
   onChanged();
 }
+if (other.hasSkipLock()) {
+  setSkipLock(other.getSkipLock());
+}
 this.mergeUnknownFields(other.getUnknownFields());
 return