hbase git commit: HBASE-17973 Expand list_regions to filter on data locality

[elserj]

Repository: hbase
Updated Branches:
  refs/heads/master 94c14ad0f -> 13b6fdf8a


HBASE-17973 Expand list_regions to filter on data locality


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/13b6fdf8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/13b6fdf8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/13b6fdf8

Branch: refs/heads/master
Commit: 13b6fdf8ad81e236632a2dc99e6c4a317213858e
Parents: 94c14ad
Author: Josh Elser 
Authored: Fri Apr 28 12:04:26 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 13:46:41 2017 -0400

--
 hbase-shell/src/main/ruby/hbase_constants.rb|  2 +
 .../main/ruby/shell/commands/list_regions.rb| 63 
 .../ruby/hbase/list_regions_test_no_cluster.rb  | 61 +++
 3 files changed, 113 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/13b6fdf8/hbase-shell/src/main/ruby/hbase_constants.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb 
b/hbase-shell/src/main/ruby/hbase_constants.rb
index c02d5c6..55ae9e7 100644
--- a/hbase-shell/src/main/ruby/hbase_constants.rb
+++ b/hbase-shell/src/main/ruby/hbase_constants.rb
@@ -81,6 +81,8 @@ module HBaseConstants
   NAMESPACES = 'NAMESPACES'
   CONFIG = 'CONFIG'
   DATA = 'DATA'
+  SERVER_NAME = 'SERVER_NAME'
+  LOCALITY_THRESHOLD = 'LOCALITY_THRESHOLD'
 
   # Load constants from hbase java API
   def self.promote_constants(constants)

http://git-wip-us.apache.org/repos/asf/hbase/blob/13b6fdf8/hbase-shell/src/main/ruby/shell/commands/list_regions.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/list_regions.rb 
b/hbase-shell/src/main/ruby/shell/commands/list_regions.rb
index 527a6cb..94b7a29 100644
--- a/hbase-shell/src/main/ruby/shell/commands/list_regions.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/list_regions.rb
@@ -23,37 +23,62 @@ module Shell
   def help
 
 return< list_regions 'table_name'
 hbase> list_regions 'table_name', 'server_name'
+hbase> list_regions 'table_name', {SERVER_NAME => 'server_name', 
LOCALITY_THRESHOLD => 0.8}
 
 EOF
 return
   end
 
-  def command(table_name, region_server_name = "")
+  def command(table_name, options = nil)
+if options.nil?
+  options = {}
+elsif not options.is_a? Hash
+  # When options isn't a hash, assume it's the server name
+  # and create the hash internally
+  options = {SERVER_NAME => options}
+end
 admin_instance = admin.instance_variable_get("@admin")
 conn_instance = admin_instance.getConnection()
 cluster_status = admin_instance.getClusterStatus()
 hregion_locator_instance = 
conn_instance.getRegionLocator(TableName.valueOf(table_name))
-hregion_locator_list = hregion_locator_instance.getAllRegionLocations()
+hregion_locator_list = 
hregion_locator_instance.getAllRegionLocations().to_a
 results = Array.new
 
 begin
-  hregion_locator_list.each do |hregion|
+  # Filter out region servers which we don't want, default to all RS
+  regions = hregion_locator_list.filter do |hregion|
+server_name = options[SERVER_NAME] || '*'
+accept_server_name? server_name, hregion.getServerName().toString()
+  end
+  # A locality threshold of "1.0" would be all regions (cannot have 
greater than 1 locality)
+  # Regions which have a `dataLocality` less-than-or-equal to this 
value are accepted
+  locality_threshold = 1.0
+  if options.has_key? LOCALITY_THRESHOLD
+value = options[LOCALITY_THRESHOLD]
+# Value validation. Must be a Float, and must be between [0, 1.0]
+raise "#{LOCALITY_THRESHOLD} must be a float value" unless 
value.is_a? Float
+raise "#{LOCALITY_THRESHOLD} must be between 0 and 1.0, inclusive" 
unless valid_locality_threshold? value
+locality_threshold = value
+  end
+  regions.each do |hregion|
 hregion_info = hregion.getRegionInfo()
 server_name = hregion.getServerName()
-if hregion.getServerName().toString.start_with? region_server_name
-  startKey = Bytes.toString(hregion.getRegionInfo().getStartKey())
-  endKey = Bytes.toString(hregion.getRegionInfo().getEndKey())
-  region_load_map = 
cluster_status.getLoad(server_name).getRegionsLoad()
-  region_load = region_load_map.get(hregion_info.getRegionName())
+region_load_map = 

hbase git commit: HBASE-17976 Remove stability annotation from public audience class

[elserj]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-16961 70bcf3fe6 -> 0d5028664


HBASE-17976 Remove stability annotation from public audience class


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d502866
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d502866
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d502866

Branch: refs/heads/HBASE-16961
Commit: 0d5028664b8f33450ed41e83362fca50bec3871d
Parents: 70bcf3f
Author: Josh Elser 
Authored: Fri Apr 28 16:51:17 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 12:58:26 2017 -0400

--
 .../java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.java  | 2 --
 1 file changed, 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0d502866/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.java
index 34d2542..023e855 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.java
@@ -17,7 +17,6 @@
 package org.apache.hadoop.hbase.quotas;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
 
 /**
  * Enumeration that represents the action HBase will take when a space quota 
is violated.
@@ -26,7 +25,6 @@ import 
org.apache.hadoop.hbase.classification.InterfaceStability;
  * namespace, it is treated as a collection of tables (all tables are subject 
to the same policy).
  */
 @InterfaceAudience.Public
-@InterfaceStability.Evolving
 public enum SpaceViolationPolicy {
   /**
* Disables the table(s).

[1/5] hbase git commit: HBASE-17968 Fix NOTICE.txt for src-release

[elserj]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 fa4f63894 -> 30c8b3802
  refs/heads/branch-1.1 e62d7a6d2 -> 8436292eb
  refs/heads/branch-1.2 236a8d06a -> 66e2aa9d9
  refs/heads/branch-1.3 e4c8a858b -> 12415f8bd
  refs/heads/master ef94de3eb -> 94c14ad0f


HBASE-17968 Fix NOTICE.txt for src-release


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94c14ad0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94c14ad0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94c14ad0

Branch: refs/heads/master
Commit: 94c14ad0f692f60b53c091c1ae705ca0ca4a35a6
Parents: ef94de3
Author: Josh Elser 
Authored: Fri Apr 28 16:39:21 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 11:25:44 2017 -0400

--
 NOTICE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/94c14ad0/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index 71efa52..7959857 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache HBase
-Copyright 2007-2015 The Apache Software Foundation
+Copyright 2007-2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

[2/5] hbase git commit: HBASE-17968 Fix NOTICE.txt for src-release

[elserj]

HBASE-17968 Fix NOTICE.txt for src-release


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/30c8b380
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/30c8b380
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/30c8b380

Branch: refs/heads/branch-1
Commit: 30c8b380208cbdbde2fb28cd07f4a5fe1901109a
Parents: fa4f638
Author: Josh Elser 
Authored: Fri Apr 28 16:39:21 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 11:27:14 2017 -0400

--
 NOTICE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/30c8b380/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index 91213d9..229e7ce 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache HBase
-Copyright 2007-2015 The Apache Software Foundation
+Copyright 2007-2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

[5/5] hbase git commit: HBASE-17968 Fix NOTICE.txt for src-release

[elserj]

HBASE-17968 Fix NOTICE.txt for src-release


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8436292e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8436292e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8436292e

Branch: refs/heads/branch-1.1
Commit: 8436292eb24b62d6b0d32b0051295a4725245b46
Parents: e62d7a6
Author: Josh Elser 
Authored: Fri Apr 28 16:39:21 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 11:28:15 2017 -0400

--
 NOTICE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8436292e/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index fb16a28..dc16d1f 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache HBase
-Copyright 2007-2015 The Apache Software Foundation
+Copyright 2007-2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

[3/5] hbase git commit: HBASE-17968 Fix NOTICE.txt for src-release

[elserj]

HBASE-17968 Fix NOTICE.txt for src-release


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/12415f8b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/12415f8b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/12415f8b

Branch: refs/heads/branch-1.3
Commit: 12415f8bde2402ce96ba1450bc73104e62e2df7c
Parents: e4c8a85
Author: Josh Elser 
Authored: Fri Apr 28 16:39:21 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 11:27:27 2017 -0400

--
 NOTICE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/12415f8b/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index 91213d9..229e7ce 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache HBase
-Copyright 2007-2015 The Apache Software Foundation
+Copyright 2007-2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

[4/5] hbase git commit: HBASE-17968 Fix NOTICE.txt for src-release

[elserj]

HBASE-17968 Fix NOTICE.txt for src-release


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66e2aa9d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66e2aa9d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66e2aa9d

Branch: refs/heads/branch-1.2
Commit: 66e2aa9d9d561f08513db498e839d7f85a520342
Parents: 236a8d0
Author: Josh Elser 
Authored: Fri Apr 28 16:39:21 2017 -0400
Committer: Josh Elser 
Committed: Mon May 1 11:27:46 2017 -0400

--
 NOTICE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/66e2aa9d/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index fb16a28..dc16d1f 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache HBase
-Copyright 2007-2015 The Apache Software Foundation
+Copyright 2007-2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

[2/4] hbase git commit: HBASE-17955 Various reviewboard improvements to space quota work

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/70bcf3fe/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index c70b736..b886f5c 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10173,42 +10173,42 @@ public final class RegionServerStatusProtos {
  * A region identifier
  * 
  *
- * optional .hbase.pb.RegionInfo region = 1;
+ * optional .hbase.pb.RegionInfo region_info = 1;
  */
-boolean hasRegion();
+boolean hasRegionInfo();
 /**
  * 
  * A region identifier
  * 
  *
- * optional .hbase.pb.RegionInfo region = 1;
+ * optional .hbase.pb.RegionInfo region_info = 1;
  */
-org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
getRegion();
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
getRegionInfo();
 /**
  * 
  * A region identifier
  * 
  *
- * optional .hbase.pb.RegionInfo region = 1;
+ * optional .hbase.pb.RegionInfo region_info = 1;
  */
-
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
 getRegionOrBuilder();
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
 getRegionInfoOrBuilder();
 
 /**
  * 
  * The size in bytes of the region
  * 
  *
- * optional uint64 size = 2;
+ * optional uint64 region_size = 2;
  */
-boolean hasSize();
+boolean hasRegionSize();
 /**
  * 
  * The size in bytes of the region
  * 
  *
- * optional uint64 size = 2;
+ * optional uint64 region_size = 2;
  */
-long getSize();
+long getRegionSize();
   }
   /**
* Protobuf type {@code hbase.pb.RegionSpaceUse}
@@ -10222,7 +10222,7 @@ public final class RegionServerStatusProtos {
   super(builder);
 }
 private RegionSpaceUse() {
-  size_ = 0L;
+  regionSize_ = 0L;
 }
 
 @java.lang.Override
@@ -10256,19 +10256,19 @@ public final class RegionServerStatusProtos {
 case 10: {
   
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
 subBuilder = null;
   if (((bitField0_ & 0x0001) == 0x0001)) {
-subBuilder = region_.toBuilder();
+subBuilder = regionInfo_.toBuilder();
   }
-  region_ = 
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
 extensionRegistry);
+  regionInfo_ = 
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
 extensionRegistry);
   if (subBuilder != null) {
-subBuilder.mergeFrom(region_);
-region_ = subBuilder.buildPartial();
+subBuilder.mergeFrom(regionInfo_);
+regionInfo_ = subBuilder.buildPartial();
   }
   bitField0_ |= 0x0001;
   break;
 }
 case 16: {
   bitField0_ |= 0x0002;
-  size_ = input.readUInt64();
+  regionSize_ = input.readUInt64();
   break;
 }
   }
@@ -10296,16 +10296,16 @@ public final class RegionServerStatusProtos {
 }
 
 private int bitField0_;
-public static final int REGION_FIELD_NUMBER = 1;
-private 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
region_;
+public static final int REGION_INFO_FIELD_NUMBER = 1;
+private 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
regionInfo_;
 /**
  * 
  * A region identifier
  * 
  *
- * optional .hbase.pb.RegionInfo region = 1;
+ * optional .hbase.pb.RegionInfo region_info = 1;
  */
-public boolean hasRegion() {
+public boolean hasRegionInfo() {
   return ((bitField0_ & 0x0001) == 0x0001);
 }
 /**
@@ -10313,32 +10313,32 @@ public final class RegionServerStatusProtos {
  * A region identifier
  * 
  *
- * optional .hbase.pb.RegionInfo region = 1;
+ * optional .hbase.pb.RegionInfo region_info = 1;
  */
-public 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
getRegion() {
-  return region_ == null ? 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance()
 : region_;
+

[1/4] hbase git commit: HBASE-17955 Various reviewboard improvements to space quota work

[elserj]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-16961 cb08814a4 -> 70bcf3fe6


http://git-wip-us.apache.org/repos/asf/hbase/blob/70bcf3fe/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java
index a313fa1..c558b26 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java
@@ -17,6 +17,7 @@
 package org.apache.hadoop.hbase.quotas;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
@@ -28,7 +29,12 @@ import 
org.apache.hadoop.hbase.regionserver.RegionServerServices;
 
 /**
  * A class to ease dealing with tables that have and do not have violation 
policies
- * being enforced in a uniform manner. Immutable.
+ * being enforced. This class is immutable, expect for {@code 
locallyCachedPolicies}.
+ *
+ * The {@code locallyCachedPolicies} are mutable given the current {@code 
activePolicies}
+ * and {@code snapshots}. It is expected that when a new instance of this 
class is
+ * instantiated, we also want to invalidate those previously cached policies 
(as they
+ * may now be invalidate if we received new quota usage information).
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -36,12 +42,23 @@ public class ActivePolicyEnforcement {
   private final Map activePolicies;
   private final Map snapshots;
   private final RegionServerServices rss;
+  private final SpaceViolationPolicyEnforcementFactory factory;
+  private final Map 
locallyCachedPolicies;
 
   public 
ActivePolicyEnforcement(Map 
activePolicies,
   Map snapshots, RegionServerServices rss) {
+this(activePolicies, snapshots, rss, 
SpaceViolationPolicyEnforcementFactory.getInstance());
+  }
+
+  public 
ActivePolicyEnforcement(Map 
activePolicies,
+  Map snapshots, RegionServerServices rss,
+  SpaceViolationPolicyEnforcementFactory factory) {
 this.activePolicies = activePolicies;
 this.snapshots = snapshots;
 this.rss = rss;
+this.factory = factory;
+// Mutable!
+this.locallyCachedPolicies = new HashMap<>();
   }
 
   /**
@@ -65,16 +82,25 @@ public class ActivePolicyEnforcement {
*/
   public SpaceViolationPolicyEnforcement getPolicyEnforcement(TableName 
tableName) {
 SpaceViolationPolicyEnforcement policy = 
activePolicies.get(Objects.requireNonNull(tableName));
-if (null == policy) {
-  synchronized (activePolicies) {
-// If we've never seen a snapshot, assume no use, and infinite limit
-SpaceQuotaSnapshot snapshot = snapshots.get(tableName);
-if (null == snapshot) {
-  snapshot = SpaceQuotaSnapshot.getNoSuchSnapshot();
+if (policy == null) {
+  synchronized (locallyCachedPolicies) {
+// When we don't have an policy enforcement for the table, there could 
be one of two cases:
+//  1) The table has no quota defined
+//  2) The table is not in violation of its quota
+// In both of these cases, we want to make sure that access remains 
fast and we minimize
+// object creation. We can accomplish this by locally caching policies 
instead of creating
+// a new instance of the policy each time.
+policy = locallyCachedPolicies.get(tableName);
+// We have already created/cached the enforcement, use it again. 
`activePolicies` and
+// `snapshots` are immutable, thus this policy is valid for the 
lifetime of `this`.
+if (policy != null) {
+  return policy;
 }
-// Create the default policy and cache it
-return 
SpaceViolationPolicyEnforcementFactory.getInstance().createWithoutViolation(
-rss, tableName, snapshot);
+// Create a PolicyEnforcement for this table and snapshot. The 
snapshot may be null
+// which is OK.
+policy = factory.createWithoutViolation(rss, tableName, 
snapshots.get(tableName));
+// Cache the policy we created
+locallyCachedPolicies.put(tableName, policy);
   }
 }
 return policy;
@@ -87,6 +113,14 @@ public class ActivePolicyEnforcement {
 return Collections.unmodifiableMap(activePolicies);
   }
 
+  /**
+   * Returns an unmodifiable version of the policy enforcements that were 
cached because they are
+   * not in violation of their quota.
+   */
+  Map 

[3/4] hbase git commit: HBASE-17955 Various reviewboard improvements to space quota work

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/70bcf3fe/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 4577bcf..e8a57e9 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
 
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
   getSpaceFieldBuilder() {
 if (spaceBuilder_ == null) {
   spaceBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6077,7 +6077,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota quota = 1;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
 
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
   getQuotaFieldBuilder() {
 if (quotaBuilder_ == null) {
   quotaBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6143,13 +6143,13 @@ public final class QuotaProtos {
   org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
 
 /**
- * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ * optional .hbase.pb.SpaceViolationPolicy violation_policy = 
1;
  */
-boolean hasPolicy();
+boolean hasViolationPolicy();
 /**
- * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ * optional .hbase.pb.SpaceViolationPolicy violation_policy = 
1;
  */
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy
 getPolicy();
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy
 getViolationPolicy();
 
 /**
  * optional bool in_violation = 2;
@@ -6163,7 +6163,7 @@ public final class QuotaProtos {
   /**
* 
* Represents the state of a quota on a table. Either the quota is not in 
violation
-   * or it is in violatino there is a violation policy which should be in 
effect.
+   * or it is in violation there is a violation policy which should be in 
effect.
* 
*
* Protobuf type {@code hbase.pb.SpaceQuotaStatus}
@@ -6177,7 +6177,7 @@ public final class QuotaProtos {
   super(builder);
 }
 private SpaceQuotaStatus() {
-  policy_ = 1;
+  violationPolicy_ = 1;
   inViolation_ = false;
 }
 
@@ -6216,7 +6216,7 @@ public final class QuotaProtos {
 unknownFields.mergeVarintField(1, rawValue);
   } else {
 bitField0_ |= 0x0001;
-policy_ = rawValue;
+violationPolicy_ = rawValue;
   }
   break;
 }
@@ -6250,19 +6250,19 @@ public final class QuotaProtos {
 }
 
 private int bitField0_;
-public static final int POLICY_FIELD_NUMBER = 1;
-private int policy_;
+public static final int VIOLATION_POLICY_FIELD_NUMBER = 1;
+private int violationPolicy_;
 /**
- * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ * optional .hbase.pb.SpaceViolationPolicy violation_policy = 
1;
  */
-public boolean hasPolicy() {
+public boolean hasViolationPolicy() {
   return ((bitField0_ & 0x0001) == 0x0001);
 }
 /**
- * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ * optional .hbase.pb.SpaceViolationPolicy violation_policy = 
1;
  */
-public 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy
 getPolicy() 

[4/4] hbase git commit: HBASE-17955 Various reviewboard improvements to space quota work

[elserj]

HBASE-17955 Various reviewboard improvements to space quota work

Most notable change is to cache SpaceViolationPolicyEnforcement objects
in the write path. When a table has no quota or there is not SpaceQuotaSnapshot
for that table (yet), we want to avoid creating lots of
SpaceViolationPolicyEnforcement instances, caching one instance
instead. This will help reduce GC pressure.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/70bcf3fe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/70bcf3fe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/70bcf3fe

Branch: refs/heads/HBASE-16961
Commit: 70bcf3fe6890582e00f9ad0ec7b6b80ebfacf05f
Parents: cb08814
Author: Josh Elser 
Authored: Tue Apr 18 16:43:40 2017 -0400
Committer: Josh Elser 
Committed: Fri Apr 28 13:27:19 2017 -0400

--
 .../hbase/quotas/QuotaSettingsFactory.java  |  10 +-
 .../hadoop/hbase/quotas/QuotaTableUtil.java |   7 +-
 .../hadoop/hbase/quotas/SpaceLimitSettings.java |  26 +-
 .../hadoop/hbase/quotas/SpaceQuotaSnapshot.java |  34 +-
 .../hbase/quotas/SpaceViolationPolicy.java  |   5 +-
 .../hbase/master/MetricsMasterQuotaSource.java  |  13 +-
 .../MetricsRegionServerQuotaSource.java |  10 +-
 .../MetricsMasterQuotaSourceFactoryImpl.java|   2 +-
 .../master/MetricsMasterQuotaSourceImpl.java|  10 +-
 .../shaded/protobuf/generated/AdminProtos.java  |   8 +-
 .../shaded/protobuf/generated/MasterProtos.java |  10 +-
 .../shaded/protobuf/generated/QuotaProtos.java  | 637 ++-
 .../generated/RegionServerStatusProtos.java | 340 +-
 .../src/main/protobuf/Quota.proto   |   8 +-
 .../src/main/protobuf/RegionServerStatus.proto  |   4 +-
 .../hbase/protobuf/generated/QuotaProtos.java   | 463 +++---
 hbase-protocol/src/main/protobuf/Quota.proto|   8 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |   4 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   9 +-
 .../hadoop/hbase/master/MetricsMaster.java  |  13 +-
 .../hbase/master/MetricsMasterWrapperImpl.java  |   4 +-
 .../hbase/quotas/ActivePolicyEnforcement.java   |  54 +-
 .../quotas/FileSystemUtilizationChore.java  |   4 +-
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   8 +-
 .../hbase/quotas/MasterSpaceQuotaObserver.java  |   4 +-
 .../quotas/NamespaceQuotaSnapshotStore.java |   2 +-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |  62 +-
 .../quotas/RegionServerSpaceQuotaManager.java   |  16 +-
 .../hbase/quotas/SpaceLimitingException.java|   6 +-
 .../hbase/quotas/SpaceQuotaRefresherChore.java  |   2 +-
 .../SpaceViolationPolicyEnforcementFactory.java |  20 +-
 .../hbase/quotas/TableQuotaSnapshotStore.java   |   2 +-
 .../AbstractViolationPolicyEnforcement.java |  45 +-
 ...LoadVerifyingViolationPolicyEnforcement.java |  50 --
 .../DefaultViolationPolicyEnforcement.java  |  90 +++
 .../DisableTableViolationPolicyEnforcement.java |   2 +-
 ...ssingSnapshotViolationPolicyEnforcement.java |  63 ++
 .../NoInsertsViolationPolicyEnforcement.java|   2 +-
 .../NoWritesViolationPolicyEnforcement.java |   2 +-
 .../hbase/regionserver/CompactSplitThread.java  |   2 +-
 .../hbase/regionserver/HRegionServer.java   |   4 +-
 .../hbase/regionserver/RSRpcServices.java   |   9 +-
 .../resources/hbase-webapps/master/table.jsp|   8 +-
 .../hbase/quotas/SpaceQuotaHelperForTests.java  |  66 +-
 .../quotas/TestActivePolicyEnforcement.java |  62 +-
 .../quotas/TestMasterSpaceQuotaObserver.java|  28 +-
 .../TestQuotaObserverChoreRegionReports.java|   6 +-
 .../TestQuotaObserverChoreWithMiniCluster.java  |  31 +-
 .../hbase/quotas/TestQuotaStatusRPCs.java   |  15 +-
 .../TestRegionServerSpaceQuotaManager.java  |   4 +-
 .../hadoop/hbase/quotas/TestSpaceQuotas.java|  30 +-
 .../TestTableSpaceQuotaViolationNotifier.java   |   8 +-
 ...kLoadCheckingViolationPolicyEnforcement.java |   2 +-
 .../TestRegionServerRegionSpaceUseReport.java   |   4 +-
 54 files changed, 1289 insertions(+), 1049 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/70bcf3fe/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 184277d..a99235f 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -127,11 +127,11 @@ public class QuotaSettingsFactory {
   }
 
   static QuotaSettings fromSpace(TableName 

[45/50] [abbrv] hbase git commit: HBASE-17003 Documentation updates for space quotas

[elserj]

HBASE-17003 Documentation updates for space quotas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/546908ed
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/546908ed
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/546908ed

Branch: refs/heads/HBASE-16961
Commit: 546908ed4b741129b8bc245c67aade1aaee29527
Parents: cb91790
Author: Josh Elser 
Authored: Thu Mar 16 16:21:14 2017 -0400
Committer: Josh Elser 
Committed: Tue Apr 25 18:37:38 2017 -0400

--
 src/main/asciidoc/_chapters/ops_mgt.adoc | 64 ++-
 1 file changed, 63 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/546908ed/src/main/asciidoc/_chapters/ops_mgt.adoc
--
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc 
b/src/main/asciidoc/_chapters/ops_mgt.adoc
index e4c077f..f9009f3 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1705,7 +1705,7 @@ handling multiple workloads:
 
 [[quota]]
 === Quotas
-HBASE-11598 introduces quotas, which allow you to throttle requests based on
+HBASE-11598 introduces RPC quotas, which allow you to throttle requests based 
on
 the following limits:
 
 . <>
@@ -1885,6 +1885,68 @@ at the same time and that fewer scans can be executed at 
the same time. A value
 `0.9` will give more queue/handlers to scans, so the number of scans executed 
will
 increase and the number of gets will decrease.
 
+[[space-quotas]]
+=== Space Quotas
+
+link:https://issues.apache.org/jira/browse/HBASE-16961[HBASE-16961] introduces 
a new type of
+quotas for HBase to leverage: filesystem quotas. These "space" quotas limit 
the amount of space
+on the filesystem that HBase namespaces and tables can consume. If a user, 
malicious or ignorant,
+has the ability to write data into HBase, with enough time, that user can 
effectively crash HBase
+(or worse HDFS) by consuming all available space. When there is no filesystem 
space available,
+HBase crashes because it can no longer create/sync data to the write-ahead log.
+
+This feature allows a for a limit to be set on the size of a table or 
namespace. When a space quota is set
+on a namespace, the quota's limit applies to the sum of usage of all tables in 
that namespace.
+When a table with a quota exists in a namespace with a quota, the table quota 
takes priority
+over the namespace quota. This allows for a scenario where a large limit can 
be placed on
+a collection of tables, but a single table in that collection can have a 
fine-grained limit set.
+
+The existing `set_quota` and `list_quota` HBase shell commands can be used to 
interact with
+space quotas. Space quotas are quotas with a `TYPE` of `SPACE` and have 
`LIMIT` and `POLICY`
+attributes. The `LIMIT` is a string that refers to the amount of space on the 
filesystem
+that the quota subject (e.g. the table or namespace) may consume. For example, 
valid values
+of `LIMIT` are `'10G'`, `'2T'`, or `'256M'`. The `POLICY` refers to the action 
that HBase will
+take when the quota subject's usage exceeds the `LIMIT`. The following are 
valid `POLICY` values.
+
+* `NO_INSERTS` - No new data may be written (e.g. `Put`, `Increment`, 
`Append`).
+* `NO_WRITES` - Same as `NO_INSERTS` but `Deletes` are also disallowed.
+* `NO_WRITES_COMPACTIONS` - Same as `NO_WRITES` but compactions are also 
disallowed.
+* `DISABLE` - The table(s) are disabled, preventing all read/write access.
+
+.Setting simple space quotas
+
+# Sets a quota on the table 't1' with a limit of 1GB, disallowing 
Puts/Increments/Appends when the table exceeds 1GB
+hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY => 
NO_INSERTS
+
+# Sets a quota on the namespace 'ns1' with a limit of 50TB, disallowing 
Puts/Increments/Appends/Deletes
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY => 
NO_WRITES
+
+# Sets a quota on the table 't3' with a limit of 2TB, disallowing any writes 
and compactions when the table exceeds 2TB.
+hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY => 
NO_WRITES_COMPACTIONS
+
+# Sets a quota on the table 't2' with a limit of 50GB, disabling the table 
when it exceeds 50GB
+hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY => 
DISABLE
+
+
+Consider the following scenario to set up quotas on a namespace, overriding 
the quota on tables in that namespace
+
+.Table and Namespace space quotas
+
+hbase> create_namespace 'ns1'
+hbase> create 'ns1:t1'
+hbase> create 'ns1:t2'
+hbase> create 'ns1:t3'
+hbase> set_quota TYPE => SPACE, NAMESPACE => 

[43/50] [abbrv] hbase git commit: HBASE-17428 Implement informational RPCs for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index a4c6095..d56def5 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
 
   getSpaceFieldBuilder() {
 if (spaceBuilder_ == null) {
   spaceBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6077,7 +6077,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota quota = 1;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
 
   getQuotaFieldBuilder() {
 if (quotaBuilder_ == null) {
   quotaBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6351,7 +6351,7 @@ public final class QuotaProtos {
 return memoizedHashCode;
   }
   int hash = 41;
-  hash = (19 * hash) + getDescriptorForType().hashCode();
+  hash = (19 * hash) + getDescriptor().hashCode();
   if (hasPolicy()) {
 hash = (37 * hash) + POLICY_FIELD_NUMBER;
 hash = (53 * hash) + policy_;
@@ -6978,7 +6978,7 @@ public final class QuotaProtos {
 return memoizedHashCode;
   }
   int hash = 41;
-  hash = (19 * hash) + getDescriptorForType().hashCode();
+  hash = (19 * hash) + getDescriptor().hashCode();
   if (hasStatus()) {
 hash = (37 * hash) + STATUS_FIELD_NUMBER;
 hash = (53 * hash) + getStatus().hashCode();
@@ -7351,7 +7351,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuotaStatus status = 1;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatusOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatusOrBuilder>
 
   getStatusFieldBuilder() {
 if (statusBuilder_ == null) {
   statusBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -7476,163 +7476,5829 @@ public final class QuotaProtos {
 
   }
 
-  private static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-internal_static_hbase_pb_TimedQuota_descriptor;
-  private static final
-
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
-  internal_static_hbase_pb_TimedQuota_fieldAccessorTable;
-  private static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-internal_static_hbase_pb_Throttle_descriptor;
-  private static final
-
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
-  internal_static_hbase_pb_Throttle_fieldAccessorTable;
-  private static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-   

[50/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas

[elserj]

HBASE-17002 JMX metrics and some UI additions for space quotas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb91790d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb91790d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb91790d

Branch: refs/heads/HBASE-16961
Commit: cb91790d2ade376c0ac895fd5b8598b7b1f76b45
Parents: d9c22cc
Author: Josh Elser 
Authored: Wed Feb 15 14:24:57 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:37:38 2017 -0400

--
 .../hbase/client/ConnectionImplementation.java  |8 +
 .../hadoop/hbase/client/QuotaStatusCalls.java   |   39 +-
 .../client/ShortCircuitMasterConnection.java|8 +
 .../hadoop/hbase/quotas/QuotaTableUtil.java |   41 +
 .../hbase/shaded/protobuf/RequestConverter.java |   11 +
 .../hbase/master/MetricsMasterQuotaSource.java  |   75 +
 .../master/MetricsMasterQuotaSourceFactory.java |   26 +
 .../hbase/master/MetricsMasterWrapper.java  |   13 +
 .../MetricsRegionServerQuotaSource.java |   54 +
 .../MetricsMasterQuotaSourceFactoryImpl.java|   36 +
 .../master/MetricsMasterQuotaSourceImpl.java|  129 +
 ...hadoop.hbase.master.MetricsMasterQuotaSource |   18 +
 ...hbase.master.MetricsMasterQuotaSourceFactory |   18 +
 .../shaded/protobuf/generated/MasterProtos.java |   93 +-
 .../shaded/protobuf/generated/QuotaProtos.java  | 3099 +-
 .../src/main/protobuf/Master.proto  |6 +-
 .../src/main/protobuf/Quota.proto   |   17 +
 .../org/apache/hadoop/hbase/master/HMaster.java |2 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   38 +
 .../hadoop/hbase/master/MetricsMaster.java  |   42 +
 .../hbase/master/MetricsMasterWrapperImpl.java  |   42 +-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |   92 +-
 .../resources/hbase-webapps/master/table.jsp|   59 +
 .../hbase/master/TestMasterMetricsWrapper.java  |   17 +
 .../hbase/quotas/TestQuotaStatusRPCs.java   |   83 +
 25 files changed, 4032 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 3022ae4..9476ce7 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -92,6 +92,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
@@ -1747,6 +1749,12 @@ class ConnectionImplementation implements 
ClusterConnection, Closeable {
   throws ServiceException {
 return stub.getSpaceQuotaRegionSizes(controller, request);
   }
+
+  @Override
+  public GetQuotaStatesResponse getQuotaStates(
+  RpcController controller, GetQuotaStatesRequest request) throws 
ServiceException {
+return stub.getQuotaStates(controller, request);
+  }
 };
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
index f0f385d..af36d1e 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 

[44/50] [abbrv] hbase git commit: HBASE-17428 Implement informational RPCs for space quotas

[elserj]

HBASE-17428 Implement informational RPCs for space quotas

Create some RPCs that can expose the in-memory state that the
RegionServers and Master hold to drive the space quota "state machine".
Then, create some hbase shell commands to interact with those.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/609333c9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/609333c9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/609333c9

Branch: refs/heads/HBASE-16961
Commit: 609333c9fcd9dae7a244cfcbc714d8614a1242b4
Parents: a714675
Author: Josh Elser 
Authored: Tue Feb 21 15:36:39 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:32:00 2017 -0400

--
 .../hbase/client/ConnectionImplementation.java  |9 +
 .../hadoop/hbase/client/QuotaStatusCalls.java   |  125 +
 .../client/ShortCircuitMasterConnection.java|7 +
 .../hadoop/hbase/quotas/QuotaTableUtil.java |   77 +
 .../hbase/shaded/protobuf/RequestConverter.java |   33 +
 .../shaded/protobuf/generated/AdminProtos.java  |  394 +-
 .../shaded/protobuf/generated/MasterProtos.java |   92 +-
 .../shaded/protobuf/generated/QuotaProtos.java  | 5986 +-
 .../generated/RegionServerStatusProtos.java |   28 +-
 .../src/main/protobuf/Admin.proto   |9 +
 .../src/main/protobuf/Master.proto  |4 +
 .../src/main/protobuf/Quota.proto   |   35 +
 .../hbase/protobuf/generated/QuotaProtos.java   |6 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   60 +
 .../hbase/quotas/ActivePolicyEnforcement.java   |8 +
 .../hbase/regionserver/RSRpcServices.java   |   57 +
 .../hadoop/hbase/master/MockRegionServer.java   |   18 +
 .../hbase/quotas/TestQuotaStatusRPCs.java   |  192 +
 hbase-shell/src/main/ruby/hbase/quotas.rb   |   16 +
 hbase-shell/src/main/ruby/shell.rb  |3 +
 .../ruby/shell/commands/list_quota_snapshots.rb |   59 +
 .../shell/commands/list_quota_table_sizes.rb|   47 +
 .../shell/commands/list_quota_violations.rb |   48 +
 hbase-shell/src/test/ruby/hbase/quotas_test.rb  |   24 -
 .../test/ruby/hbase/quotas_test_no_cluster.rb   |   69 +
 25 files changed, 7086 insertions(+), 320 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 6859cb3..3022ae4 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -92,6 +92,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.DisableReplicationPeerRequest;
@@ -1738,6 +1740,13 @@ class ConnectionImplementation implements 
ClusterConnection, Closeable {
   ListReplicationPeersRequest request) throws ServiceException {
 return stub.listReplicationPeers(controller, request);
   }
+
+  @Override
+  public GetSpaceQuotaRegionSizesResponse getSpaceQuotaRegionSizes(
+  RpcController controller, GetSpaceQuotaRegionSizesRequest request)
+  throws ServiceException {
+return stub.getSpaceQuotaRegionSizes(controller, request);
+  }
 };
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
new file mode 100644
index 000..f0f385d
--- 

[40/50] [abbrv] hbase git commit: HBASE-17602 Reduce some quota chore periods/delays

[elserj]

HBASE-17602 Reduce some quota chore periods/delays


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/986a642a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/986a642a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/986a642a

Branch: refs/heads/HBASE-16961
Commit: 986a642a2d5cf23d52f80f031103abe60ab4af13
Parents: 359e36c
Author: Josh Elser 
Authored: Tue Feb 7 11:21:08 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:32:00 2017 -0400

--
 .../java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java  | 4 ++--
 .../org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/986a642a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index b9f4592..7f894e4 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -55,11 +55,11 @@ public class QuotaObserverChore extends ScheduledChore {
   private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
   static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
   "hbase.master.quotas.observer.chore.period";
-  static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5 
minutes in millis
+  static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1 
minutes in millis
 
   static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
   "hbase.master.quotas.observer.chore.delay";
-  static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
+  static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15 
seconds in millis
 
   static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
   "hbase.master.quotas.observer.chore.timeunit";

http://git-wip-us.apache.org/repos/asf/hbase/blob/986a642a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
index e1a2693..8587e79 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
@@ -44,11 +44,11 @@ public class SpaceQuotaRefresherChore extends 
ScheduledChore {
 
   static final String POLICY_REFRESHER_CHORE_PERIOD_KEY =
   "hbase.regionserver.quotas.policy.refresher.chore.period";
-  static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5 
minutes in millis
+  static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1 
minute in millis
 
   static final String POLICY_REFRESHER_CHORE_DELAY_KEY =
   "hbase.regionserver.quotas.policy.refresher.chore.delay";
-  static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
+  static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15 
seconds in millis
 
   static final String POLICY_REFRESHER_CHORE_TIMEUNIT_KEY =
   "hbase.regionserver.quotas.policy.refresher.chore.timeunit";

[49/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index d56def5..4577bcf 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -13024,6 +13024,3031 @@ public final class QuotaProtos {
 
   }
 
+  public interface GetQuotaStatesRequestOrBuilder extends
+  // 
@@protoc_insertion_point(interface_extends:hbase.pb.GetQuotaStatesRequest)
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+  }
+  /**
+   * Protobuf type {@code hbase.pb.GetQuotaStatesRequest}
+   */
+  public  static final class GetQuotaStatesRequest extends
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3 
implements
+  // 
@@protoc_insertion_point(message_implements:hbase.pb.GetQuotaStatesRequest)
+  GetQuotaStatesRequestOrBuilder {
+// Use GetQuotaStatesRequest.newBuilder() to construct.
+private 
GetQuotaStatesRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
 builder) {
+  super(builder);
+}
+private GetQuotaStatesRequest() {
+}
+
+@java.lang.Override
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private GetQuotaStatesRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream 
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite 
extensionRegistry)
+throws 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 {
+  this();
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder 
unknownFields =
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+  }
+}
+  } catch 
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_descriptor;
+}
+
+protected 
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.class,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+  byte isInitialized = memoizedIsInitialized;
+  if (isInitialized == 1) return true;
+  if (isInitialized == 0) return false;
+
+  memoizedIsInitialized = 1;
+  return true;
+}
+
+public void 
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream 
output)
+throws java.io.IOException {
+  unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+  int size = memoizedSize;
+  if (size != -1) return size;
+
+  size = 0;
+  size += unknownFields.getSerializedSize();
+  memoizedSize = size;
+  return size;
+}
+
+private static final long serialVersionUID = 0L;
+@java.lang.Override
+public boolean equals(final java.lang.Object obj) {
+  if (obj == this) {
+   return true;
+ 

[46/50] [abbrv] hbase git commit: HBASE-17447 Implement a MasterObserver for automatically deleting space quotas

[elserj]

HBASE-17447 Implement a MasterObserver for automatically deleting space quotas

When a table or namespace is deleted, it would be nice to automatically
delete the quota on said table/NS. It's possible that not all people
would want this functionality so we can leave it up to the user to
configure this Observer.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb08814a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb08814a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb08814a

Branch: refs/heads/HBASE-16961
Commit: cb08814a409c5b69f7f5e74c0f0aab2cc0d77890
Parents: d09d967
Author: Josh Elser 
Authored: Thu Mar 16 18:54:01 2017 -0400
Committer: Josh Elser 
Committed: Tue Apr 25 18:37:38 2017 -0400

--
 .../hbase/quotas/MasterSpaceQuotaObserver.java  |  85 ++
 .../quotas/TestMasterSpaceQuotaObserver.java| 169 +++
 src/main/asciidoc/_chapters/ops_mgt.adoc|  17 ++
 3 files changed, 271 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cb08814a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
new file mode 100644
index 000..a3abf32
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+
+/**
+ * An observer to automatically delete space quotas when a table/namespace
+ * are deleted.
+ */
+@InterfaceAudience.Private
+public class MasterSpaceQuotaObserver implements MasterObserver {
+  private CoprocessorEnvironment cpEnv;
+  private Configuration conf;
+  private boolean quotasEnabled = false;
+
+  @Override
+  public void start(CoprocessorEnvironment ctx) throws IOException {
+this.cpEnv = ctx;
+this.conf = cpEnv.getConfiguration();
+this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf);
+  }
+
+  @Override
+  public void postDeleteTable(
+  ObserverContext ctx, TableName tableName) 
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+  return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+if (null != quotas && quotas.hasSpace()) {
+  QuotaSettings settings = 
QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+  try (Admin admin = conn.getAdmin()) {
+admin.setQuota(settings);
+  }
+}
+  }
+
+  @Override
+  public void postDeleteNamespace(
+  ObserverContext ctx, String namespace) 
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+  return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(), 
namespace);
+if (null != quotas && quotas.hasSpace()) {
+  

[48/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-protocol-shaded/src/main/protobuf/Master.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto 
b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index ada9b35..0d0fb93 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -941,7 +941,11 @@ service MasterService {
   rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
 returns(RemoveDrainFromRegionServersResponse);
 
-  /** Fetches the Master's view of space quotas */
+  /** Fetches the Master's view of space utilization */
   rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
 returns(GetSpaceQuotaRegionSizesResponse);
+
+  /** Fetches the Master's view of quotas */
+  rpc GetQuotaStates(GetQuotaStatesRequest)
+returns(GetQuotaStatesResponse);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto 
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 2d7e5f5..1a6d5ed 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
   message RegionSizes {
 optional TableName table_name = 1;
 optional uint64 size = 2;
+
   }
   repeated RegionSizes sizes = 1;
 }
@@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
   }
   repeated TableViolationPolicy violation_policies = 1;
 }
+
+message GetQuotaStatesRequest {
+}
+
+message GetQuotaStatesResponse {
+  message TableQuotaSnapshot {
+optional TableName table_name = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+  }
+  message NamespaceQuotaSnapshot {
+optional string namespace = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+  }
+  repeated TableQuotaSnapshot table_snapshots = 1;
+  repeated NamespaceQuotaSnapshot ns_snapshots = 2;
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 128dc18..64960ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -911,7 +911,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
   // Create the quota snapshot notifier
   spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
   spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
-  this.quotaObserverChore = new QuotaObserverChore(this);
+  this.quotaObserverChore = new QuotaObserverChore(this, 
getMasterMetrics());
   // Start the chore to read the region FS space reports and act on them
   getChoreService().scheduleChore(quotaObserverChore);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index ddc58bb..faffb3f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -62,7 +62,9 @@ import 
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.LockInfo;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
 import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -113,8 +115,12 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;

[21/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages - addendum fixes line lengths (Josh Elser)

[elserj]

HBASE-16995 Build client Java API and client protobuf messages - addendum fixes 
line lengths (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a5087ed
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a5087ed
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a5087ed

Branch: refs/heads/HBASE-16961
Commit: 3a5087ed113c48ddd33a96b47c23bf62b5cceaa3
Parents: 2b9b00b
Author: tedyu 
Authored: Mon Nov 21 13:00:27 2016 -0800
Committer: Josh Elser 
Committed: Tue Apr 25 18:20:14 2017 -0400

--
 .../hbase/quotas/QuotaSettingsFactory.java  | 20 
 .../hadoop/hbase/quotas/SpaceLimitSettings.java |  8 
 .../hbase/shaded/protobuf/ProtobufUtil.java |  7 ---
 3 files changed, 20 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3a5087ed/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 8512e39..7f1c180 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -128,7 +128,8 @@ public class QuotaSettingsFactory {
 
   static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota 
protoQuota) {
 if ((null == table && null == namespace) || (null != table && null != 
namespace)) {
-  throw new IllegalArgumentException("Can only construct 
SpaceLimitSettings for a table or namespace.");
+  throw new IllegalArgumentException(
+  "Can only construct SpaceLimitSettings for a table or namespace.");
 }
 if (null != table) {
   return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
@@ -300,29 +301,32 @@ public class QuotaSettingsFactory {
*/
 
   /**
-   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given table to the given size in bytes.
-   * When the space usage is exceeded by the table, the provided {@link 
SpaceViolationPolicy} is enacted on the table.
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given table
+   * to the given size in bytes. When the space usage is exceeded by the 
table, the provided
+   * {@link SpaceViolationPolicy} is enacted on the table.
*
* @param tableName The name of the table on which the quota should be 
applied.
* @param sizeLimit The limit of a table's size in bytes.
* @param violationPolicy The action to take when the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
-  public static QuotaSettings limitTableSpace(final TableName tableName, long 
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+  public static QuotaSettings limitTableSpace(
+  final TableName tableName, long sizeLimit, final SpaceViolationPolicy 
violationPolicy) {
 return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
   }
 
   /**
-   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given namespace to the given size in bytes.
-   * When the space usage is exceeded by all tables in the namespace, the 
provided {@link SpaceViolationPolicy} is enacted on
-   * all tables in the namespace.
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given
+   * namespace to the given size in bytes. When the space usage is exceeded by 
all tables in the
+   * namespace, the provided {@link SpaceViolationPolicy} is enacted on all 
tables in the namespace.
*
* @param namespace The namespace on which the quota should be applied.
* @param sizeLimit The limit of the namespace's size in bytes.
* @param violationPolicy The action to take when the the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
-  public static QuotaSettings limitNamespaceSpace(final String namespace, long 
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+  public static QuotaSettings limitNamespaceSpace(
+  final String namespace, long sizeLimit, final SpaceViolationPolicy 
violationPolicy) {
 return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3a5087ed/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java

[15/50] [abbrv] hbase git commit: HBASE-15143 Procedure v2 - Web UI displaying queues

[elserj]

HBASE-15143 Procedure v2 - Web UI displaying queues

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/25575064
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/25575064
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/25575064

Branch: refs/heads/HBASE-16961
Commit: 25575064154fe1cc7ff8970e8f15a3cff648f37a
Parents: 1367519
Author: Balazs Meszaros 
Authored: Mon Feb 13 13:50:56 2017 -0800
Committer: Michael Stack 
Committed: Tue Apr 25 09:39:28 2017 -0700

--
 .../org/apache/hadoop/hbase/client/Admin.java   |9 +
 .../hbase/client/ConnectionImplementation.java  |   11 +-
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |   42 +-
 .../client/ShortCircuitMasterConnection.java|6 +
 .../hbase/shaded/protobuf/ProtobufUtil.java |  183 +-
 .../hadoop/hbase/procedure2/LockInfo.java   |  128 +
 .../hadoop/hbase/procedure2/LockAndQueue.java   |   21 +-
 .../hadoop/hbase/procedure2/LockStatus.java |1 +
 .../hbase/procedure2/ProcedureScheduler.java|7 +
 .../hadoop/hbase/procedure2/ProcedureUtil.java  |4 +-
 .../procedure2/SimpleProcedureScheduler.java|   10 +-
 .../protobuf/generated/LockServiceProtos.java   | 2423 +-
 .../shaded/protobuf/generated/MasterProtos.java | 2152 
 .../src/main/protobuf/LockService.proto |   22 +
 .../src/main/protobuf/Master.proto  |   11 +
 .../hbase/tmpl/master/MasterStatusTmpl.jamon|2 +-
 .../hbase/coprocessor/MasterObserver.java   |   19 +
 .../org/apache/hadoop/hbase/master/HMaster.java |   37 +-
 .../hbase/master/MasterCoprocessorHost.java |   21 +
 .../hadoop/hbase/master/MasterRpcServices.java  |  147 +-
 .../hadoop/hbase/master/MasterServices.java |9 +-
 .../hbase/master/locking/LockProcedure.java |8 +-
 .../procedure/MasterProcedureScheduler.java |  119 +-
 .../hbase-webapps/master/procedures.jsp |  127 +-
 .../resources/hbase-webapps/master/snapshot.jsp |2 +-
 .../hbase-webapps/master/snapshotsStats.jsp |2 +-
 .../resources/hbase-webapps/master/table.jsp|2 +-
 .../hbase-webapps/master/tablesDetailed.jsp |2 +-
 .../main/resources/hbase-webapps/master/zk.jsp  |2 +-
 .../hbase/coprocessor/TestMasterObserver.java   |   38 +
 .../hbase/master/MockNoopMasterServices.java|9 +-
 .../procedure/TestMasterProcedureScheduler.java |  169 +-
 .../hadoop/hbase/protobuf/TestProtobufUtil.java |   41 +-
 .../hbase/shaded/protobuf/TestProtobufUtil.java |  151 ++
 hbase-shell/src/main/ruby/hbase/admin.rb|5 +
 hbase-shell/src/main/ruby/shell.rb  |3 +-
 hbase-shell/src/main/ruby/shell/commands.rb |5 +
 .../src/main/ruby/shell/commands/list_locks.rb  |   60 +
 hbase-shell/src/main/ruby/shell/formatter.rb|9 +-
 .../src/test/ruby/shell/list_locks_test.rb  |  152 ++
 40 files changed, 5409 insertions(+), 762 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index f2fc9a5..3e767d2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -45,6 +45,7 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.replication.TableCFs;
 import org.apache.hadoop.hbase.client.security.SecurityCapability;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.hadoop.hbase.procedure2.LockInfo;
 import org.apache.hadoop.hbase.quotas.QuotaFilter;
 import org.apache.hadoop.hbase.quotas.QuotaRetriever;
 import org.apache.hadoop.hbase.quotas.QuotaSettings;
@@ -1250,6 +1251,14 @@ public interface Admin extends Abortable, Closeable {
   throws IOException;
 
   /**
+   * List locks.
+   * @return lock list
+   * @throws IOException if a remote or network exception occurs
+   */
+  LockInfo[] listLocks()
+  throws IOException;
+
+  /**
* Roll the log writer. I.e. for filesystem based write ahead logs, start 
writing to a new file.
*
* Note that the actual rolling of the log writer is asynchronous and may 
not be complete when

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git 

[19/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages - addendum fixes white spaces (Josh Elser)

[elserj]

HBASE-16995 Build client Java API and client protobuf messages - addendum fixes 
white spaces (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b9b00b8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b9b00b8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b9b00b8

Branch: refs/heads/HBASE-16961
Commit: 2b9b00b835bc163302b68152fbb1feace4896b9b
Parents: 23194dc
Author: tedyu 
Authored: Thu Nov 17 10:42:18 2016 -0800
Committer: Josh Elser 
Committed: Tue Apr 25 18:19:48 2017 -0400

--
 .../hbase/quotas/TestQuotaSettingsFactory.java|  2 +-
 .../shaded/protobuf/generated/MasterProtos.java   |  2 +-
 .../shaded/protobuf/generated/QuotaProtos.java| 18 +-
 .../hbase/protobuf/generated/QuotaProtos.java |  4 ++--
 4 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2b9b00b8/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
index 17015d6..e0012a7 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
@@ -44,7 +44,7 @@ import org.junit.experimental.categories.Category;
  */
 @Category(SmallTests.class)
 public class TestQuotaSettingsFactory {
-  
+
   @Test
   public void testAllQuotasAddedToList() {
 final SpaceQuota spaceQuota = SpaceQuota.newBuilder()

http://git-wip-us.apache.org/repos/asf/hbase/blob/2b9b00b8/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index da6c65e..42ab969 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -64879,7 +64879,7 @@ public final class MasterProtos {
* optional .hbase.pb.SpaceLimitRequest space_limit = 8;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
 
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
   getSpaceLimitFieldBuilder() {
 if (spaceLimitBuilder_ == null) {
   spaceLimitBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<

http://git-wip-us.apache.org/repos/asf/hbase/blob/2b9b00b8/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index e3c6bfd..0ab2576 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
 
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 

[42/50] [abbrv] hbase git commit: HBASE-17428 Implement informational RPCs for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index e90c934..c70b736 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10429,7 +10429,7 @@ public final class RegionServerStatusProtos {
 return memoizedHashCode;
   }
   int hash = 41;
-  hash = (19 * hash) + getDescriptorForType().hashCode();
+  hash = (19 * hash) + getDescriptor().hashCode();
   if (hasRegion()) {
 hash = (37 * hash) + REGION_FIELD_NUMBER;
 hash = (53 * hash) + getRegion().hashCode();
@@ -10824,7 +10824,7 @@ public final class RegionServerStatusProtos {
* optional .hbase.pb.RegionInfo region = 1;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo, 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo, 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
 
   getRegionFieldBuilder() {
 if (regionBuilder_ == null) {
   regionBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -10940,7 +10940,7 @@ public final class RegionServerStatusProtos {
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
  */
-
java.util.List
+
java.util.List
 
 getSpaceUseList();
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
@@ -10953,7 +10953,7 @@ public final class RegionServerStatusProtos {
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
  */
-java.util.List
+java.util.List
 
 getSpaceUseOrBuilderList();
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
@@ -11056,7 +11056,7 @@ public final class RegionServerStatusProtos {
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
  */
-public java.util.List
+public java.util.List
 
 getSpaceUseOrBuilderList() {
   return spaceUse_;
 }
@@ -11142,7 +11142,7 @@ public final class RegionServerStatusProtos {
 return memoizedHashCode;
   }
   int hash = 41;
-  hash = (19 * hash) + getDescriptorForType().hashCode();
+  hash = (19 * hash) + getDescriptor().hashCode();
   if (getSpaceUseCount() > 0) {
 hash = (37 * hash) + SPACE_USE_FIELD_NUMBER;
 hash = (53 * hash) + getSpaceUseList().hashCode();
@@ -11368,7 +11368,7 @@ public final class RegionServerStatusProtos {
   spaceUseBuilder_ = null;
   spaceUse_ = other.spaceUse_;
   bitField0_ = (bitField0_ & ~0x0001);
-  spaceUseBuilder_ =
+  spaceUseBuilder_ = 
 
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
 ?
getSpaceUseFieldBuilder() : null;
 } else {
@@ -11604,7 +11604,7 @@ public final class RegionServerStatusProtos {
   /**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-  public java.util.List
+  public java.util.List
 
getSpaceUseOrBuilderList() {
 if (spaceUseBuilder_ != null) {
   return spaceUseBuilder_.getMessageOrBuilderList();
@@ -11630,12 +11630,12 @@ public final class RegionServerStatusProtos {
   /**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-  public 
java.util.List
+  public 
java.util.List
 
getSpaceUseBuilderList() {
 return getSpaceUseFieldBuilder().getBuilderList();
   }
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.RepeatedFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse,
 

[30/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 8b127d9..973ac8c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -37,9 +37,8 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -54,51 +53,51 @@ import com.google.common.collect.Multimap;
 @InterfaceAudience.Private
 public class QuotaObserverChore extends ScheduledChore {
   private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
-  static final String VIOLATION_OBSERVER_CHORE_PERIOD_KEY =
-  "hbase.master.quotas.violation.observer.chore.period";
-  static final int VIOLATION_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 
5 minutes in millis
+  static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
+  "hbase.master.quotas.observer.chore.period";
+  static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5 
minutes in millis
 
-  static final String VIOLATION_OBSERVER_CHORE_DELAY_KEY =
-  "hbase.master.quotas.violation.observer.chore.delay";
-  static final long VIOLATION_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
+  static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
+  "hbase.master.quotas.observer.chore.delay";
+  static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
 
-  static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_KEY =
-  "hbase.master.quotas.violation.observer.chore.timeunit";
-  static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_DEFAULT = 
TimeUnit.MILLISECONDS.name();
+  static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
+  "hbase.master.quotas.observer.chore.timeunit";
+  static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT = 
TimeUnit.MILLISECONDS.name();
 
-  static final String VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY =
-  "hbase.master.quotas.violation.observer.report.percent";
-  static final double VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
+  static final String QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY =
+  "hbase.master.quotas.observer.report.percent";
+  static final double QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
 
   private final Connection conn;
   private final Configuration conf;
   private final MasterQuotaManager quotaManager;
   /*
-   * Callback that changes in quota violation are passed to.
+   * Callback that changes in quota snapshots are passed to.
*/
-  private final SpaceQuotaViolationNotifier violationNotifier;
+  private final SpaceQuotaSnapshotNotifier snapshotNotifier;
 
   /*
-   * Preserves the state of quota violations for tables and namespaces
+   * Preserves the state of quota snapshots for tables and namespaces
*/
-  private final Map tableQuotaViolationStates;
-  private final Map namespaceQuotaViolationStates;
+  private final Map tableQuotaSnapshots;
+  private final Map namespaceQuotaSnapshots;
 
   /*
-   * Encapsulates logic for moving tables/namespaces into or out of quota 
violation
+   * Encapsulates logic for tracking the state of a table/namespace WRT space 
quotas
*/
-  private QuotaViolationStore tableViolationStore;
-  private QuotaViolationStore namespaceViolationStore;
+  private QuotaSnapshotStore tableSnapshotStore;
+  private QuotaSnapshotStore namespaceSnapshotStore;
 
   public QuotaObserverChore(HMaster master) {
 this(
 master.getConnection(), master.getConfiguration(),
-master.getSpaceQuotaViolationNotifier(), 
master.getMasterQuotaManager(),
+master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
 master);
   }
 
   QuotaObserverChore(
-  Connection conn, Configuration conf, SpaceQuotaViolationNotifier 
violationNotifier,
+  Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier 
snapshotNotifier,
   

[34/50] [abbrv] hbase git commit: HBASE-17025 Add shell commands for space quotas

[elserj]

HBASE-17025 Add shell commands for space quotas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a35c425
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a35c425
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a35c425

Branch: refs/heads/HBASE-16961
Commit: 3a35c425c6fae27a736fb7d25631b27025de3732
Parents: 67f7c8c
Author: Josh Elser 
Authored: Wed Jan 11 11:55:29 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:24:20 2017 -0400

--
 hbase-shell/src/main/ruby/hbase/quotas.rb   |  62 -
 hbase-shell/src/main/ruby/hbase_constants.rb|   1 +
 .../src/main/ruby/shell/commands/set_quota.rb   |  45 +-
 .../hadoop/hbase/client/AbstractTestShell.java  |   1 +
 hbase-shell/src/test/ruby/hbase/quotas_test.rb  | 137 +++
 hbase-shell/src/test/ruby/tests_runner.rb   |   1 +
 6 files changed, 242 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3a35c425/hbase-shell/src/main/ruby/hbase/quotas.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb 
b/hbase-shell/src/main/ruby/hbase/quotas.rb
index bf2dc63..d99fe72 100644
--- a/hbase-shell/src/main/ruby/hbase/quotas.rb
+++ b/hbase-shell/src/main/ruby/hbase/quotas.rb
@@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
 java_import org.apache.hadoop.hbase.quotas.QuotaFilter
 java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
 java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
+java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
 
 module HBaseQuotasConstants
+  # RPC Quota constants
   GLOBAL_BYPASS = 'GLOBAL_BYPASS'
   THROTTLE_TYPE = 'THROTTLE_TYPE'
   THROTTLE = 'THROTTLE'
   REQUEST = 'REQUEST'
   WRITE = 'WRITE'
   READ = 'READ'
+  # Space quota constants
+  SPACE = 'SPACE'
+  NO_INSERTS = 'NO_INSERTS'
+  NO_WRITES = 'NO_WRITES'
+  NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
+  DISABLE = 'DISABLE'
 end
 
 module Hbase
@@ -107,6 +115,54 @@ module Hbase
   @admin.setQuota(settings)
 end
 
+def limit_space(args)
+  raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? 
and args.kind_of?(Hash))
+  # Let the user provide a raw number
+  if args[LIMIT].is_a?(Numeric)
+limit = args[LIMIT]
+  else
+# Parse a string a 1K, 2G, etc.
+limit = _parse_size(args[LIMIT])
+  end
+  # Extract the policy, failing if something bogus was provided
+  policy = SpaceViolationPolicy.valueOf(args[POLICY])
+  # Create a table or namespace quota
+  if args.key?(TABLE)
+if args.key?(NAMESPACE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+settings = 
QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)), 
limit, policy)
+  elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+settings = 
QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy)
+  else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+  end
+  # Apply the quota
+  @admin.setQuota(settings)
+end
+
+def remove_space_limit(args)
+  raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? 
and args.kind_of?(Hash))
+  if args.key?(TABLE)
+if args.key?(NAMESPACE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+table = TableName.valueOf(args.delete(TABLE))
+settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
+  elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+settings = 
QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
+  else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+  end
+  @admin.setQuota(settings)
+end
+
 def set_global_bypass(bypass, args)
   raise(ArgumentError, "Arguments should be a Hash") unless 
args.kind_of?(Hash)
 
@@ -171,7 +227,7 @@ module Hbase
   return _size_from_str(match[1].to_i, match[2])
 end
   else
-raise "Invalid size limit syntax"
+raise(ArgumentError, "Invalid size limit syntax")
   end
 end
 
@@ -188,7 +244,7 @@ module Hbase
 end
 
 if limit <= 0
-  raise "Invalid throttle limit, must be greater then 0"
+  raise(ArgumentError, "Invalid throttle limit, must be greater then 
0")
 end
 
 

[24/50] [abbrv] hbase git commit: HBASE-16998 Implement Master-side analysis of region space reports

[elserj]

HBASE-16998 Implement Master-side analysis of region space reports

Adds a new Chore to the Master that analyzes the reports that are
sent by RegionServers. The Master must then, for all tables with
quotas, determine the tables that are violating quotas and move
those tables into violation. Similarly, tables no longer violating
the quota can be moved out of violation.

The Chore is the "stateful" bit, managing which tables are and
are not in violation. Everything else is just performing
computation and informing the Chore on the updated state.

Added InterfaceAudience annotations and clean up the QuotaObserverChore
constructor. Cleaned up some javadoc and QuotaObserverChore. Reuse
the QuotaViolationStore impl objects.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f592b00
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f592b00
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f592b00

Branch: refs/heads/HBASE-16961
Commit: 5f592b000572597b17601cc12286f38b4719b4d7
Parents: cf7dd95
Author: Josh Elser 
Authored: Tue Nov 8 18:55:12 2016 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:24:20 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaRetriever.java |  27 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  20 +
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   1 +
 .../quotas/NamespaceQuotaViolationStore.java| 127 
 .../hadoop/hbase/quotas/QuotaObserverChore.java | 618 +++
 .../hbase/quotas/QuotaViolationStore.java   |  89 +++
 .../quotas/SpaceQuotaViolationNotifier.java |  44 ++
 .../SpaceQuotaViolationNotifierForTest.java |  50 ++
 .../hbase/quotas/TableQuotaViolationStore.java  | 127 
 .../TestNamespaceQuotaViolationStore.java   | 156 +
 .../hbase/quotas/TestQuotaObserverChore.java| 106 
 .../TestQuotaObserverChoreWithMiniCluster.java  | 596 ++
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |   4 -
 .../quotas/TestTableQuotaViolationStore.java| 151 +
 .../hbase/quotas/TestTablesWithQuotas.java  | 198 ++
 15 files changed, 2305 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5f592b00/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
index 0f7baa5..4482693 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.LinkedList;
+import java.util.Objects;
 import java.util.Queue;
 
 import org.apache.commons.logging.Log;
@@ -54,11 +55,23 @@ public class QuotaRetriever implements Closeable, 
Iterable {
   private Connection connection;
   private Table table;
 
-  private QuotaRetriever() {
+  /**
+   * Should QutoaRetriever manage the state of the connection, or leave it be.
+   */
+  private boolean isManagedConnection = false;
+
+  QuotaRetriever() {
   }
 
   void init(final Configuration conf, final Scan scan) throws IOException {
-this.connection = ConnectionFactory.createConnection(conf);
+// Set this before creating the connection and passing it down to make sure
+// it's cleaned up if we fail to construct the Scanner.
+this.isManagedConnection = true;
+init(ConnectionFactory.createConnection(conf), scan);
+  }
+
+  void init(final Connection conn, final Scan scan) throws IOException {
+this.connection = Objects.requireNonNull(conn);
 this.table = this.connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
 try {
   scanner = table.getScanner(scan);
@@ -77,10 +90,14 @@ public class QuotaRetriever implements Closeable, 
Iterable {
   this.table.close();
   this.table = null;
 }
-if (this.connection != null) {
-  this.connection.close();
-  this.connection = null;
+// Null out the connection on close() even if we didn't explicitly close it
+// to maintain typical semantics.
+if (isManagedConnection) {
+  if (this.connection != null) {
+this.connection.close();
+  }
 }
+this.connection = null;
   }
 
   public QuotaSettings next() throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/5f592b00/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 

[10/50] [abbrv] hbase git commit: HBASE-17952 The new options for PE tool do not work

[elserj]

HBASE-17952 The new options for PE tool do not work


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72fac379
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72fac379
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72fac379

Branch: refs/heads/HBASE-16961
Commit: 72fac379815d79fdeac1452b36cd12fb5492f627
Parents: 49f707f
Author: zhangduo 
Authored: Mon Apr 24 17:00:32 2017 +0800
Committer: zhangduo 
Committed: Tue Apr 25 09:34:49 2017 +0800

--
 .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java  | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/72fac379/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index a3d3254..d0b7319 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -685,6 +685,9 @@ public class PerformanceEvaluation extends Configured 
implements Tool {
   this.columns = that.columns;
   this.caching = that.caching;
   this.inMemoryCompaction = that.inMemoryCompaction;
+  this.asyncPrefetch = that.asyncPrefetch;
+  this.cacheBlocks = that.cacheBlocks;
+  this.scanReadType = that.scanReadType;
 }
 
 public int getCaching() {

[23/50] [abbrv] hbase git commit: HBASE-16998 Implement Master-side analysis of region space reports

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/5f592b00/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 000..98236c2
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+  private static final Log LOG = 
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+  private static final int SIZE_PER_VALUE = 256;
+  private static final String F1 = "f1";
+  private static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+  private static final AtomicLong COUNTER = new AtomicLong(0);
+  private static final long ONE_MEGABYTE = 1024L * 1024L;
+  private static final long DEFAULT_WAIT_MILLIS = 500;
+
+  @Rule
+  public TestName testName = new TestName();
+
+  private HMaster master;
+  private QuotaObserverChore chore;
+  private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 
1000);
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 
1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void removeAllQuotas() throws Exception {
+final Connection conn = TEST_UTIL.getConnection();
+

[25/50] [abbrv] hbase git commit: HBASE-16999 Implement master and regionserver synchronization of quota state

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/cfd374d0/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
new file mode 100644
index 000..4a7000c
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Test case for {@link TableSpaceQuotaViolationNotifier}.
+ */
+@Category(SmallTests.class)
+public class TestTableSpaceQuotaViolationNotifier {
+
+  private TableSpaceQuotaViolationNotifier notifier;
+  private Connection conn;
+
+  @Before
+  public void setup() throws Exception {
+notifier = new TableSpaceQuotaViolationNotifier();
+conn = mock(Connection.class);
+notifier.initialize(conn);
+  }
+
+  @Test
+  public void testToViolation() throws Exception {
+final TableName tn = TableName.valueOf("inviolation");
+final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Put expectedPut = new Put(Bytes.toBytes("t." + 
tn.getNameAsString()));
+final SpaceQuota protoQuota = SpaceQuota.newBuilder()
+.setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(policy))
+.build();
+expectedPut.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"), 
protoQuota.toByteArray());
+
+notifier.transitionTableToViolation(tn, policy);
+
+verify(quotaTable).put(argThat(new SingleCellPutMatcher(expectedPut)));
+  }
+
+  @Test
+  public void testToObservance() throws Exception {
+final TableName tn = TableName.valueOf("notinviolation");
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Delete expectedDelete = new Delete(Bytes.toBytes("t." + 
tn.getNameAsString()));
+expectedDelete.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"));
+
+notifier.transitionTableToObservance(tn);
+
+verify(quotaTable).delete(argThat(new 
SingleCellDeleteMatcher(expectedDelete)));
+  }
+
+  /**
+   * Parameterized for Puts.
+   */
+  private static class SingleCellPutMatcher extends 
SingleCellMutationMatcher {
+private SingleCellPutMatcher(Put expected) {
+  super(expected);
+}
+  }
+
+  /**
+   * Parameterized for Deletes.
+   */
+  private static class SingleCellDeleteMatcher extends 
SingleCellMutationMatcher {
+private SingleCellDeleteMatcher(Delete expected) {
+  super(expected);
+}
+  }
+
+  /**
+   * Quick hack to verify a Mutation with one column.
+   */
+  private static class SingleCellMutationMatcher extends ArgumentMatcher 
{
+private final Mutation expected;
+
+private SingleCellMutationMatcher(Mutation expected) {
+  

[26/50] [abbrv] hbase git commit: HBASE-16999 Implement master and regionserver synchronization of quota state

[elserj]

HBASE-16999 Implement master and regionserver synchronization of quota state

* Implement the RegionServer reading violation from the quota table
* Implement the Master reporting violations to the quota table
* RegionServers need to track its enforced policies


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cfd374d0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cfd374d0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cfd374d0

Branch: refs/heads/HBASE-16961
Commit: cfd374d04f008e2e2890f800692b176c143f32bb
Parents: 5f592b0
Author: Josh Elser 
Authored: Fri Nov 18 15:38:19 2016 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:24:20 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  92 -
 .../org/apache/hadoop/hbase/master/HMaster.java |  35 +++-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |   5 +-
 .../hbase/quotas/RegionServerQuotaManager.java  | 200 ---
 .../quotas/RegionServerRpcQuotaManager.java | 200 +++
 .../quotas/RegionServerSpaceQuotaManager.java   | 169 
 .../quotas/SpaceQuotaViolationNotifier.java |  16 +-
 .../SpaceQuotaViolationNotifierFactory.java |  62 ++
 .../SpaceQuotaViolationNotifierForTest.java |   4 +
 ...SpaceQuotaViolationPolicyRefresherChore.java | 154 ++
 .../TableSpaceQuotaViolationNotifier.java   |  55 +
 .../hbase/regionserver/HRegionServer.java   |  21 +-
 .../hbase/regionserver/RSRpcServices.java   |   7 +-
 .../regionserver/RegionServerServices.java  |  12 +-
 .../hadoop/hbase/MockRegionServerServices.java  |  10 +-
 .../hadoop/hbase/master/MockRegionServer.java   |  10 +-
 .../TestQuotaObserverChoreWithMiniCluster.java  |   2 +
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |  47 +
 .../hadoop/hbase/quotas/TestQuotaThrottle.java  |   4 +-
 .../TestRegionServerSpaceQuotaManager.java  | 127 
 ...SpaceQuotaViolationPolicyRefresherChore.java | 131 
 .../TestTableSpaceQuotaViolationNotifier.java   | 144 +
 22 files changed, 1281 insertions(+), 226 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cfd374d0/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 8ef4f08..b5eac48 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -24,16 +24,20 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
@@ -44,7 +48,12 @@ import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.filter.RegexStringComparator;
 import org.apache.hadoop.hbase.filter.RowFilter;
 import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
+import 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Strings;
 
@@ -53,9 +62,8 @@ import org.apache.hadoop.hbase.util.Strings;
  * 
  * ROW-KEY  FAM/QUALDATA
  *   n.namespace q:s global-quotas
- *   n.namespace u:dusize in bytes
  *   t.table q:s global-quotas
- *   t.table u:dusize in bytes
+ *   t.table u:vspace violation policy
  *   u.user  q:s global-quotas
  

[16/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/23194dca/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index 05894b9..1925828 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -217,12 +217,20 @@ public final class QuotaProtos {
  * THROTTLE = 1;
  */
 THROTTLE(0, 1),
+/**
+ * SPACE = 2;
+ */
+SPACE(1, 2),
 ;
 
 /**
  * THROTTLE = 1;
  */
 public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
 
 
 public final int getNumber() { return value; }
@@ -230,6 +238,7 @@ public final class QuotaProtos {
 public static QuotaType valueOf(int value) {
   switch (value) {
 case 1: return THROTTLE;
+case 2: return SPACE;
 default: return null;
   }
 }
@@ -281,6 +290,142 @@ public final class QuotaProtos {
 // @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
   }
 
+  /**
+   * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+   *
+   * 
+   * Defines what action should be taken when the SpaceQuota is violated
+   * 
+   */
+  public enum SpaceViolationPolicy
+  implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * DISABLE = 1;
+ *
+ * 
+ * Disable the table(s)
+ * 
+ */
+DISABLE(0, 1),
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ */
+NO_WRITES_COMPACTIONS(1, 2),
+/**
+ * NO_WRITES = 3;
+ *
+ * 
+ * No writes or bulk-loads
+ * 
+ */
+NO_WRITES(2, 3),
+/**
+ * NO_INSERTS = 4;
+ *
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ */
+NO_INSERTS(3, 4),
+;
+
+/**
+ * DISABLE = 1;
+ *
+ * 
+ * Disable the table(s)
+ * 
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * NO_WRITES = 3;
+ *
+ * 
+ * No writes or bulk-loads
+ * 
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * NO_INSERTS = 4;
+ *
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() { return value; }
+
+public static SpaceViolationPolicy valueOf(int value) {
+  switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+  }
+}
+
+public static 
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+  return internalValueMap;
+}
+private static 
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+  new com.google.protobuf.Internal.EnumLiteMap() 
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+  return SpaceViolationPolicy.valueOf(number);
+}
+  };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+  return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+  return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+  if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+  "EnumValueDescriptor is not for this type.");
+  }
+  return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private SpaceViolationPolicy(int index, int value) {
+  this.index = index;
+  this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.SpaceViolationPolicy)
+  }
+
   public interface TimedQuotaOrBuilder
   extends com.google.protobuf.MessageOrBuilder {
 
@@ -3315,6 +3460,20 @@ public final class QuotaProtos {
 

[38/50] [abbrv] hbase git commit: HBASE-17000 Implement computation of online region sizes and report to the Master

[elserj]

HBASE-17000 Implement computation of online region sizes and report to the 
Master

Includes a trivial implementation of the Master-side collection to
avoid. Only enough to write a test to verify RS collection.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/91b27eda
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/91b27eda
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/91b27eda

Branch: refs/heads/HBASE-16961
Commit: 91b27eda65c438836cd4e3166a9cd683485275a8
Parents: e0c0e95
Author: Josh Elser 
Authored: Mon Nov 7 13:46:42 2016 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:24:20 2017 -0400

--
 .../generated/RegionServerStatusProtos.java | 2071 +-
 .../src/main/protobuf/RegionServerStatus.proto  |   22 +
 .../hadoop/hbase/master/MasterRpcServices.java  |   19 +
 .../quotas/FileSystemUtilizationChore.java  |  205 ++
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   15 +
 .../hbase/regionserver/HRegionServer.java   |   72 +
 .../quotas/TestFileSystemUtilizationChore.java  |  357 +++
 .../hadoop/hbase/quotas/TestRegionSizeUse.java  |  194 ++
 .../TestRegionServerRegionSpaceUseReport.java   |   99 +
 9 files changed, 3032 insertions(+), 22 deletions(-)
--

[06/50] [abbrv] hbase git commit: HBASE-17943 Addendum increases the threshold value of in-memory compaction for TestWalAndCompactingMemStoreFlush

[elserj]

HBASE-17943 Addendum increases the threshold value of in-memory compaction for 
TestWalAndCompactingMemStoreFlush


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9053ec6f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9053ec6f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9053ec6f

Branch: refs/heads/HBASE-16961
Commit: 9053ec6fe6505eba4f14adfdd83329511e4a77f0
Parents: e95cf47
Author: Chia-Ping Tsai 
Authored: Sat Apr 22 20:47:55 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Sat Apr 22 20:47:55 2017 +0800

--
 .../hbase/regionserver/TestWalAndCompactingMemStoreFlush.java   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9053ec6f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
index 2c16399..3b2ebe2 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
@@ -36,7 +36,6 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -136,7 +135,7 @@ public class TestWalAndCompactingMemStoreFlush {
 conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY,
 FlushNonSloppyStoresFirstPolicy.class.getName());
 
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
 75 * 1024);
-conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 
0.25);
+conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 
0.5);
 // set memstore to do data compaction
 conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
 String.valueOf(MemoryCompactionPolicy.EAGER));
@@ -771,7 +770,7 @@ public class TestWalAndCompactingMemStoreFlush {
 FlushNonSloppyStoresFirstPolicy.class.getName());
 
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
 75 * 1024);
-conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 
0.5);
+conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 
0.8);
 // set memstore to do index compaction with merge
 conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
 String.valueOf(MemoryCompactionPolicy.BASIC));

[02/50] [abbrv] hbase git commit: HBASE-17864: Implement async snapshot/cloneSnapshot/restoreSnapshot methods

[elserj]

HBASE-17864: Implement async snapshot/cloneSnapshot/restoreSnapshot methods

Signed-off-by: Guanghao Zhang 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d39f40e7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d39f40e7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d39f40e7

Branch: refs/heads/HBASE-16961
Commit: d39f40e787ecab54ee597ac4463bbbd2f5e944d9
Parents: 33dadc1
Author: huzheng 
Authored: Thu Apr 20 18:59:43 2017 +0800
Committer: Guanghao Zhang 
Committed: Fri Apr 21 18:57:43 2017 +0800

--
 .../apache/hadoop/hbase/client/AsyncAdmin.java  |  88 +++
 .../hadoop/hbase/client/AsyncHBaseAdmin.java| 145 +++
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |   3 +-
 .../org/apache/hadoop/hbase/HConstants.java |   4 +
 .../hbase/client/TestAsyncSnapshotAdminApi.java | 112 ++
 5 files changed, 351 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d39f40e7/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 5d2955f..b7c60dd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -573,4 +573,92 @@ public interface AsyncAdmin {
* {@link CompletableFuture}.
*/
   CompletableFuture listReplicatedTableCFs();
+
+  /**
+   * Take a snapshot for the given table. If the table is enabled, a 
FLUSH-type snapshot will be
+   * taken. If the table is disabled, an offline snapshot is taken. Snapshots 
are considered unique
+   * based on the name of the snapshot. Attempts to take a snapshot 
with the same name (even
+   * a different type or with different parameters) will fail with a
+   * {@link org.apache.hadoop.hbase.snapshot.SnapshotCreationException} 
indicating the duplicate
+   * naming. Snapshot names follow the same naming constraints as tables in 
HBase. See
+   * {@link 
org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * @param snapshotName name of the snapshot to be created
+   * @param tableName name of the table for which snapshot is created
+   */
+  CompletableFuture snapshot(String snapshotName, TableName tableName);
+
+  /**
+   * Create typed snapshot of the table. Snapshots are considered unique based 
on the name of the
+   * snapshot. Attempts to take a snapshot with the same name (even a 
different type or with
+   * different parameters) will fail with a
+   * {@link org.apache.hadoop.hbase.snapshot.SnapshotCreationException} 
indicating the duplicate
+   * naming. Snapshot names follow the same naming constraints as tables in 
HBase. See
+   * {@link 
org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * @param snapshotName name to give the snapshot on the filesystem. Must be 
unique from all other
+   *  snapshots stored on the cluster
+   * @param tableName name of the table to snapshot
+   * @param type type of snapshot to take
+   */
+  CompletableFuture snapshot(final String snapshotName, final TableName 
tableName,
+  SnapshotType type);
+
+  /**
+   * Take a snapshot and wait for the server to complete that snapshot 
asynchronously. Only a single
+   * snapshot should be taken at a time for an instance of HBase, or results 
may be undefined (you
+   * can tell multiple HBase clusters to snapshot at the same time, but only 
one at a time for a
+   * single cluster). Snapshots are considered unique based on the name of 
the snapshot.
+   * Attempts to take a snapshot with the same name (even a different type or 
with different
+   * parameters) will fail with a {@link 
org.apache.hadoop.hbase.snapshot.SnapshotCreationException}
+   * indicating the duplicate naming. Snapshot names follow the same naming 
constraints as tables in
+   * HBase. See {@link 
org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+   * You should probably use {@link #snapshot(String, 
org.apache.hadoop.hbase.TableName)} unless you
+   * are sure about the type of snapshot that you want to take.
+   * @param snapshot snapshot to take
+   */
+  CompletableFuture snapshot(SnapshotDescription snapshot);
+
+  /**
+   * Check the current state of the passed snapshot. There are three possible 
states:
+   * 
+   * running - returns false
+   * finished - returns true
+   * finished with error - throws the exception that caused the snapshot 
to fail
+   * 
+   * The cluster only knows 

[07/50] [abbrv] hbase git commit: HBASE-17302 The region flush request disappeared from flushQueue - addendum (Guangxu Cheng)

[elserj]

HBASE-17302 The region flush request disappeared from flushQueue - addendum 
(Guangxu Cheng)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/435104af
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/435104af
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/435104af

Branch: refs/heads/HBASE-16961
Commit: 435104af70232076145df4211da297c9235cd58f
Parents: 9053ec6
Author: tedyu 
Authored: Sun Apr 23 07:37:34 2017 -0700
Committer: tedyu 
Committed: Sun Apr 23 07:37:34 2017 -0700

--
 .../org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/435104af/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index aaa9572..2f2a4cf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -816,8 +816,8 @@ class MemStoreFlusher implements FlushRequester {
 return false;
   }
   FlushRegionEntry other = (FlushRegionEntry) obj;
-  if (!this.region.getRegionInfo().getRegionNameAsString().equals(
-  other.region.getRegionInfo().getRegionNameAsString())) {
+  if (!Bytes.equals(this.region.getRegionInfo().getRegionName(),
+  other.region.getRegionInfo().getRegionName())) {
 return false;
   }
   return compareTo(other) == 0;

[32/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

HBASE-17001 Enforce quota violation policies in the RegionServer

The nuts-and-bolts of filesystem quotas. The Master must inform
RegionServers of the violation of a quota by a table. The RegionServer
must apply the violation policy as configured. Need to ensure
that the proper interfaces exist to satisfy all necessary policies.

This required a massive rewrite of the internal tracking by
the general space quota feature. Instead of tracking "violations",
we need to start tracking "usage". This allows us to make the decision
at the RegionServer level as to when the files in a bulk load request
should be accept or rejected which ultimately lets us avoid bulk loads
dramatically exceeding a configured space quota.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66cf4809
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66cf4809
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66cf4809

Branch: refs/heads/HBASE-16961
Commit: 66cf48095cfcfbc04bf34873e909b1e60b24fb5e
Parents: cfd374d
Author: Josh Elser 
Authored: Thu Dec 15 13:27:56 2016 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:24:20 2017 -0400

--
 .../hbase/quotas/QuotaExceededException.java|4 +
 .../hadoop/hbase/quotas/QuotaTableUtil.java |   47 +-
 .../hadoop/hbase/quotas/SpaceQuotaSnapshot.java |  192 +++
 .../shaded/protobuf/generated/QuotaProtos.java  | 1384 +-
 .../src/main/protobuf/Quota.proto   |   15 +
 .../hbase/protobuf/generated/QuotaProtos.java   | 1324 -
 hbase-protocol/src/main/protobuf/Quota.proto|   15 +
 .../org/apache/hadoop/hbase/master/HMaster.java |   29 +-
 .../hbase/quotas/ActivePolicyEnforcement.java   |   86 ++
 .../quotas/NamespaceQuotaSnapshotStore.java |  127 ++
 .../quotas/NamespaceQuotaViolationStore.java|  127 --
 .../hadoop/hbase/quotas/QuotaObserverChore.java |  344 +++--
 .../hadoop/hbase/quotas/QuotaSnapshotStore.java |   96 ++
 .../hbase/quotas/QuotaViolationStore.java   |   89 --
 .../quotas/RegionServerSpaceQuotaManager.java   |  179 ++-
 .../hbase/quotas/SpaceLimitingException.java|   95 ++
 .../hbase/quotas/SpaceQuotaRefresherChore.java  |  225 +++
 .../quotas/SpaceQuotaSnapshotNotifier.java  |   45 +
 .../SpaceQuotaSnapshotNotifierFactory.java  |   62 +
 .../quotas/SpaceQuotaViolationNotifier.java |   54 -
 .../SpaceQuotaViolationNotifierFactory.java |   62 -
 .../SpaceQuotaViolationNotifierForTest.java |   54 -
 ...SpaceQuotaViolationPolicyRefresherChore.java |  154 --
 .../quotas/SpaceViolationPolicyEnforcement.java |   91 ++
 .../SpaceViolationPolicyEnforcementFactory.java |   95 ++
 .../hbase/quotas/TableQuotaSnapshotStore.java   |  127 ++
 .../hbase/quotas/TableQuotaViolationStore.java  |  127 --
 .../quotas/TableSpaceQuotaSnapshotNotifier.java |   52 +
 .../TableSpaceQuotaViolationNotifier.java   |   55 -
 .../AbstractViolationPolicyEnforcement.java |  118 ++
 ...LoadVerifyingViolationPolicyEnforcement.java |   50 +
 .../DisableTableViolationPolicyEnforcement.java |   80 +
 .../NoInsertsViolationPolicyEnforcement.java|   55 +
 ...esCompactionsViolationPolicyEnforcement.java |   64 +
 .../NoWritesViolationPolicyEnforcement.java |   54 +
 .../hbase/regionserver/CompactSplitThread.java  |   12 +
 .../hbase/regionserver/RSRpcServices.java   |   92 +-
 .../hbase/quotas/SpaceQuotaHelperForTests.java  |  228 +++
 .../SpaceQuotaSnapshotNotifierForTest.java  |   55 +
 .../quotas/TestActivePolicyEnforcement.java |   74 +
 .../quotas/TestFileSystemUtilizationChore.java  |3 +-
 .../TestNamespaceQuotaViolationStore.java   |   16 +-
 .../hbase/quotas/TestQuotaObserverChore.java|   30 +-
 .../TestQuotaObserverChoreWithMiniCluster.java  |  351 ++---
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |   34 +-
 .../TestRegionServerSpaceQuotaManager.java  |  174 ++-
 ...SpaceQuotaViolationPolicyRefresherChore.java |  193 ++-
 .../hadoop/hbase/quotas/TestSpaceQuotas.java|  452 ++
 .../quotas/TestTableQuotaViolationStore.java|   22 +-
 .../TestTableSpaceQuotaViolationNotifier.java   |   48 +-
 .../hbase/quotas/TestTablesWithQuotas.java  |8 +-
 .../BaseViolationPolicyEnforcement.java |   31 +
 ...kLoadCheckingViolationPolicyEnforcement.java |  142 ++
 ...tDisableTableViolationPolicyEnforcement.java |   59 +
 ...TestNoInsertsViolationPolicyEnforcement.java |   57 +
 ...esCompactionsViolationPolicyEnforcement.java |   58 +
 .../TestNoWritesViolationPolicyEnforcement.java |   57 +
 57 files changed, 6492 insertions(+), 1481 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaExceededException.java

[18/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)

[elserj]

HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23194dca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23194dca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23194dca

Branch: refs/heads/HBASE-16961
Commit: 23194dcac8b4e4dbe7598fd51fa89f18b77a59dc
Parents: 2557506
Author: tedyu 
Authored: Thu Nov 17 10:19:52 2016 -0800
Committer: Josh Elser 
Committed: Tue Apr 25 18:19:48 2017 -0400

--
 .../hbase/quotas/QuotaSettingsFactory.java  |   47 +
 .../apache/hadoop/hbase/quotas/QuotaType.java   |1 +
 .../hadoop/hbase/quotas/SpaceLimitSettings.java |  166 ++
 .../hbase/quotas/SpaceViolationPolicy.java  |   44 +
 .../hbase/shaded/protobuf/ProtobufUtil.java |   51 +
 .../hbase/quotas/TestQuotaSettingsFactory.java  |  148 ++
 .../hbase/quotas/TestSpaceLimitSettings.java|  119 ++
 .../shaded/protobuf/generated/MasterProtos.java |  588 --
 .../shaded/protobuf/generated/QuotaProtos.java  | 1739 +-
 .../src/main/protobuf/Master.proto  |2 +
 .../src/main/protobuf/Quota.proto   |   21 +
 .../hbase/protobuf/generated/QuotaProtos.java   | 1682 -
 hbase-protocol/src/main/protobuf/Quota.proto|   21 +
 13 files changed, 4293 insertions(+), 336 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/23194dca/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 3622a32..8512e39 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -27,6 +27,7 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRe
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 
 @InterfaceAudience.Public
 public class QuotaSettingsFactory {
@@ -89,6 +90,9 @@ public class QuotaSettingsFactory {
 if (quotas.getBypassGlobals() == true) {
   settings.add(new QuotaGlobalsSettingsBypass(userName, tableName, 
namespace, true));
 }
+if (quotas.hasSpace()) {
+  settings.add(fromSpace(tableName, namespace, quotas.getSpace()));
+}
 return settings;
   }
 
@@ -122,6 +126,18 @@ public class QuotaSettingsFactory {
 return settings;
   }
 
+  static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota 
protoQuota) {
+if ((null == table && null == namespace) || (null != table && null != 
namespace)) {
+  throw new IllegalArgumentException("Can only construct 
SpaceLimitSettings for a table or namespace.");
+}
+if (null != table) {
+  return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
+} else {
+  // namespace must be non-null
+  return SpaceLimitSettings.fromSpaceQuota(namespace, protoQuota);
+}
+  }
+
   /* ==
*  RPC Throttle
*/
@@ -278,4 +294,35 @@ public class QuotaSettingsFactory {
   public static QuotaSettings bypassGlobals(final String userName, final 
boolean bypassGlobals) {
 return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals);
   }
+
+  /* ==
+   *  FileSystem Space Settings
+   */
+
+  /**
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given table to the given size in bytes.
+   * When the space usage is exceeded by the table, the provided {@link 
SpaceViolationPolicy} is enacted on the table.
+   *
+   * @param tableName The name of the table on which the quota should be 
applied.
+   * @param sizeLimit The limit of a table's size in bytes.
+   * @param violationPolicy The action to take when the quota is exceeded.
+   * @return An {@link QuotaSettings} object.
+   */
+  public static QuotaSettings limitTableSpace(final TableName tableName, long 
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
+  }
+
+  /**
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given namespace to the given size in bytes.
+  

[05/50] [abbrv] hbase git commit: HBASE-16314 Retry on table snapshot failure during full backup (Vladimir Rodionov)

[elserj]

HBASE-16314 Retry on table snapshot failure during full backup (Vladimir 
Rodionov)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e95cf479
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e95cf479
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e95cf479

Branch: refs/heads/HBASE-16961
Commit: e95cf479c7615ae160a6ba963cc7689f3b440efd
Parents: a3b6f4a
Author: tedyu 
Authored: Fri Apr 21 16:15:07 2017 -0700
Committer: tedyu 
Committed: Fri Apr 21 16:15:07 2017 -0700

--
 .../hbase/backup/BackupRestoreConstants.java| 10 ++
 .../backup/impl/FullTableBackupClient.java  | 36 ++--
 2 files changed, 44 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e95cf479/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
index e46904b..d1ab246 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
@@ -37,6 +37,16 @@ public interface BackupRestoreConstants {
   public final static int BACKUP_SYSTEM_TTL_DEFAULT = HConstants.FOREVER;
   public final static String BACKUP_ENABLE_KEY = "hbase.backup.enable";
   public final static boolean BACKUP_ENABLE_DEFAULT = false;
+
+
+  public static final String BACKUP_MAX_ATTEMPTS_KEY = 
"hbase.backup.attempts.max";
+  public static final int DEFAULT_BACKUP_MAX_ATTEMPTS = 10;
+
+  public static final String BACKUP_ATTEMPTS_PAUSE_MS_KEY = 
"hbase.backup.attempts.pause.ms";
+  public static final int DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS = 1;
+
+
+
   /*
*  Drivers option list
*/

http://git-wip-us.apache.org/repos/asf/hbase/blob/e95cf479/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
index 77d1184..ee7a841 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.hbase.backup.impl;
 
+import static 
org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_ATTEMPTS_PAUSE_MS_KEY;
+import static 
org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_MAX_ATTEMPTS_KEY;
+import static 
org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS;
+import static 
org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_MAX_ATTEMPTS;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
@@ -148,8 +153,7 @@ public class FullTableBackupClient extends 
TableBackupClient {
 "snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime()) 
+ "_"
 + tableName.getNamespaceAsString() + "_" + 
tableName.getQualifierAsString();
 
-admin.snapshot(snapshotName, tableName);
-
+snapshotTable(admin, tableName, snapshotName);
 backupInfo.setSnapshotName(tableName, snapshotName);
   }
 
@@ -186,4 +190,32 @@ public class FullTableBackupClient extends 
TableBackupClient {
 
   }
 
+  private void snapshotTable(Admin admin, TableName tableName, String 
snapshotName)
+  throws IOException {
+
+int maxAttempts =
+conf.getInt(BACKUP_MAX_ATTEMPTS_KEY, DEFAULT_BACKUP_MAX_ATTEMPTS);
+int pause =
+conf.getInt(BACKUP_ATTEMPTS_PAUSE_MS_KEY, 
DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS);
+int attempts = 0;
+
+while (attempts++ < maxAttempts) {
+  try {
+admin.snapshot(snapshotName, tableName);
+return;
+  } catch (IOException ee) {
+LOG.warn("Snapshot attempt " + attempts + " failed for table " + 
tableName
++ ", sleeping for " + pause + "ms", ee);
+if (attempts < maxAttempts) {
+  try {
+Thread.sleep(pause);
+  } catch (InterruptedException e) {
+Thread.currentThread().interrupt();
+break;
+  }
+}
+  }
+}
+throw new IOException("Failed to snapshot table "+ tableName);
+  }
 }

[37/50] [abbrv] hbase git commit: HBASE-17000 Implement computation of online region sizes and report to the Master

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index d7d4db0..e90c934 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10164,6 +10164,1912 @@ public final class RegionServerStatusProtos {
 
   }
 
+  public interface RegionSpaceUseOrBuilder extends
+  // @@protoc_insertion_point(interface_extends:hbase.pb.RegionSpaceUse)
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ * 
+ * A region identifier
+ * 
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+boolean hasRegion();
+/**
+ * 
+ * A region identifier
+ * 
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
getRegion();
+/**
+ * 
+ * A region identifier
+ * 
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
 getRegionOrBuilder();
+
+/**
+ * 
+ * The size in bytes of the region
+ * 
+ *
+ * optional uint64 size = 2;
+ */
+boolean hasSize();
+/**
+ * 
+ * The size in bytes of the region
+ * 
+ *
+ * optional uint64 size = 2;
+ */
+long getSize();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.RegionSpaceUse}
+   */
+  public  static final class RegionSpaceUse extends
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3 
implements
+  // @@protoc_insertion_point(message_implements:hbase.pb.RegionSpaceUse)
+  RegionSpaceUseOrBuilder {
+// Use RegionSpaceUse.newBuilder() to construct.
+private 
RegionSpaceUse(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
 builder) {
+  super(builder);
+}
+private RegionSpaceUse() {
+  size_ = 0L;
+}
+
+@java.lang.Override
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private RegionSpaceUse(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream 
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite 
extensionRegistry)
+throws 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 {
+  this();
+  int mutable_bitField0_ = 0;
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder 
unknownFields =
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 10: {
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
 subBuilder = null;
+  if (((bitField0_ & 0x0001) == 0x0001)) {
+subBuilder = region_.toBuilder();
+  }
+  region_ = 
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
 extensionRegistry);
+  if (subBuilder != null) {
+subBuilder.mergeFrom(region_);
+region_ = subBuilder.buildPartial();
+  }
+  bitField0_ |= 0x0001;
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  size_ = input.readUInt64();
+  break;
+}
+  }
+}
+  } catch 
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+

[28/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index c493b25..943c898 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -22,16 +22,12 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
@@ -40,20 +36,15 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -62,7 +53,6 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
 
-import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Multimap;
 
@@ -72,11 +62,8 @@ import com.google.common.collect.Multimap;
 @Category(LargeTests.class)
 public class TestQuotaObserverChoreWithMiniCluster {
   private static final Log LOG = 
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
-  private static final int SIZE_PER_VALUE = 256;
-  private static final String F1 = "f1";
   private static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
   private static final AtomicLong COUNTER = new AtomicLong(0);
-  private static final long ONE_MEGABYTE = 1024L * 1024L;
   private static final long DEFAULT_WAIT_MILLIS = 500;
 
   @Rule
@@ -84,18 +71,19 @@ public class TestQuotaObserverChoreWithMiniCluster {
 
   private HMaster master;
   private QuotaObserverChore chore;
-  private SpaceQuotaViolationNotifierForTest violationNotifier;
+  private SpaceQuotaSnapshotNotifierForTest snapshotNotifier;
+  private SpaceQuotaHelperForTests helper;
 
   @BeforeClass
   public static void setUp() throws Exception {
 Configuration conf = TEST_UTIL.getConfiguration();
 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 
1000);
 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 
1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
-conf.setClass(SpaceQuotaViolationNotifierFactory.VIOLATION_NOTIFIER_KEY,
-SpaceQuotaViolationNotifierForTest.class, 
SpaceQuotaViolationNotifier.class);
+conf.setClass(SpaceQuotaSnapshotNotifierFactory.SNAPSHOT_NOTIFIER_KEY,
+SpaceQuotaSnapshotNotifierForTest.class, 
SpaceQuotaSnapshotNotifier.class);
 TEST_UTIL.startMiniCluster(1);
   }
 
@@ -131,40 +119,55 @@ public class TestQuotaObserverChoreWithMiniCluster {
 }
 
 master = TEST_UTIL.getMiniHBaseCluster().getMaster();
-violationNotifier =
-(SpaceQuotaViolationNotifierForTest) 
master.getSpaceQuotaViolationNotifier();
-violationNotifier.clearTableViolations();
+snapshotNotifier =
+(SpaceQuotaSnapshotNotifierForTest) 
master.getSpaceQuotaSnapshotNotifier();
+snapshotNotifier.clearSnapshots();
 chore = master.getQuotaObserverChore();
+

[29/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
new file mode 100644
index 000..6b754b9
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import 
org.apache.hadoop.hbase.quotas.policies.BulkLoadVerifyingViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.DisableTableViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.NoInsertsViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.NoWritesCompactionsViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.NoWritesViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * A factory class for instantiating {@link SpaceViolationPolicyEnforcement} 
instances.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SpaceViolationPolicyEnforcementFactory {
+
+  private static final SpaceViolationPolicyEnforcementFactory INSTANCE =
+  new SpaceViolationPolicyEnforcementFactory();
+
+  private SpaceViolationPolicyEnforcementFactory() {}
+
+  /**
+   * Returns an instance of this factory.
+   */
+  public static SpaceViolationPolicyEnforcementFactory getInstance() {
+return INSTANCE;
+  }
+
+  /**
+   * Constructs the appropriate {@link SpaceViolationPolicyEnforcement} for 
tables that are
+   * in violation of their space quota.
+   */
+  public SpaceViolationPolicyEnforcement create(
+  RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot 
snapshot) {
+SpaceViolationPolicyEnforcement enforcement;
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (!status.isInViolation()) {
+  throw new IllegalArgumentException(tableName + " is not in violation. 
Snapshot=" + snapshot);
+}
+switch (status.getPolicy()) {
+  case DISABLE:
+enforcement = new DisableTableViolationPolicyEnforcement();
+break;
+  case NO_WRITES_COMPACTIONS:
+enforcement = new NoWritesCompactionsViolationPolicyEnforcement();
+break;
+  case NO_WRITES:
+enforcement = new NoWritesViolationPolicyEnforcement();
+break;
+  case NO_INSERTS:
+enforcement = new NoInsertsViolationPolicyEnforcement();
+break;
+  default:
+throw new IllegalArgumentException("Unhandled SpaceViolationPolicy: " 
+ status.getPolicy());
+}
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+  }
+
+  /**
+   * Creates the "default" {@link SpaceViolationPolicyEnforcement} for a table 
that isn't in
+   * violation. This is used to have uniform policy checking for tables in and 
not quotas.
+   */
+  public SpaceViolationPolicyEnforcement createWithoutViolation(
+  RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot 
snapshot) {
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (status.isInViolation()) {
+  throw new IllegalArgumentException(
+  tableName + " is in violation. Logic error. Snapshot=" + snapshot);
+}
+BulkLoadVerifyingViolationPolicyEnforcement enforcement = new 
BulkLoadVerifyingViolationPolicyEnforcement();
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java

[36/50] [abbrv] hbase git commit: HBASE-17000 Implement computation of online region sizes and report to the Master

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto 
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 1c373ee..23ddd43 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -141,6 +141,22 @@ message SplitTableRegionResponse {
   optional uint64 proc_id = 1;
 }
 
+message RegionSpaceUse {
+  optional RegionInfo region = 1; // A region identifier
+  optional uint64 size = 2; // The size in bytes of the region
+}
+
+/**
+ * Reports filesystem usage for regions.
+ */
+message RegionSpaceUseReportRequest {
+  repeated RegionSpaceUse space_use = 1;
+}
+
+message RegionSpaceUseReportResponse {
+
+}
+
 service RegionServerStatusService {
   /** Called when a region server first starts. */
   rpc RegionServerStartup(RegionServerStartupRequest)
@@ -182,4 +198,10 @@ service RegionServerStatusService {
*/
   rpc getProcedureResult(GetProcedureResultRequest)
 returns(GetProcedureResultResponse);
+
+  /**
+   * Reports Region filesystem space use
+   */
+  rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
+returns(RegionSpaceUseReportResponse);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 40c4a71..4ee8f25 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.LockInfo;
 import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -95,6 +96,9 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
@@ -1901,4 +1905,19 @@ public class MasterRpcServices extends RSRpcServices
   throw new ServiceException(e);
 }
   }
+
+  @Override
+  public RegionSpaceUseReportResponse reportRegionSpaceUse(RpcController 
controller,
+  RegionSpaceUseReportRequest request) throws ServiceException {
+try {
+  master.checkInitialized();
+  MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+  for (RegionSpaceUse report : request.getSpaceUseList()) {
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize());
+  }
+  return RegionSpaceUseReportResponse.newBuilder().build();
+} catch (Exception e) {
+  throw new ServiceException(e);
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
new file mode 100644
index 000..01540eb
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * 

[14/50] [abbrv] hbase git commit: HBASE-15143 Procedure v2 - Web UI displaying queues

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
index 6dbf9b2..99853a5 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
@@ -104,6 +104,114 @@ public final class LockServiceProtos {
 // @@protoc_insertion_point(enum_scope:hbase.pb.LockType)
   }
 
+  /**
+   * Protobuf enum {@code hbase.pb.ResourceType}
+   */
+  public enum ResourceType
+  implements 
org.apache.hadoop.hbase.shaded.com.google.protobuf.ProtocolMessageEnum {
+/**
+ * RESOURCE_TYPE_SERVER = 1;
+ */
+RESOURCE_TYPE_SERVER(1),
+/**
+ * RESOURCE_TYPE_NAMESPACE = 2;
+ */
+RESOURCE_TYPE_NAMESPACE(2),
+/**
+ * RESOURCE_TYPE_TABLE = 3;
+ */
+RESOURCE_TYPE_TABLE(3),
+/**
+ * RESOURCE_TYPE_REGION = 4;
+ */
+RESOURCE_TYPE_REGION(4),
+;
+
+/**
+ * RESOURCE_TYPE_SERVER = 1;
+ */
+public static final int RESOURCE_TYPE_SERVER_VALUE = 1;
+/**
+ * RESOURCE_TYPE_NAMESPACE = 2;
+ */
+public static final int RESOURCE_TYPE_NAMESPACE_VALUE = 2;
+/**
+ * RESOURCE_TYPE_TABLE = 3;
+ */
+public static final int RESOURCE_TYPE_TABLE_VALUE = 3;
+/**
+ * RESOURCE_TYPE_REGION = 4;
+ */
+public static final int RESOURCE_TYPE_REGION_VALUE = 4;
+
+
+public final int getNumber() {
+  return value;
+}
+
+/**
+ * @deprecated Use {@link #forNumber(int)} instead.
+ */
+@java.lang.Deprecated
+public static ResourceType valueOf(int value) {
+  return forNumber(value);
+}
+
+public static ResourceType forNumber(int value) {
+  switch (value) {
+case 1: return RESOURCE_TYPE_SERVER;
+case 2: return RESOURCE_TYPE_NAMESPACE;
+case 3: return RESOURCE_TYPE_TABLE;
+case 4: return RESOURCE_TYPE_REGION;
+default: return null;
+  }
+}
+
+public static 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+  return internalValueMap;
+}
+private static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap<
+ResourceType> internalValueMap =
+  new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap()
 {
+public ResourceType findValueByNumber(int number) {
+  return ResourceType.forNumber(number);
+}
+  };
+
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+  return getDescriptor().getValues().get(ordinal());
+}
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+  return getDescriptor();
+}
+public static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.getDescriptor().getEnumTypes().get(1);
+}
+
+private static final ResourceType[] VALUES = values();
+
+public static ResourceType valueOf(
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
 desc) {
+  if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+  "EnumValueDescriptor is not for this type.");
+  }
+  return VALUES[desc.getIndex()];
+}
+
+private final int value;
+
+private ResourceType(int value) {
+  this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.ResourceType)
+  }
+
   public interface LockRequestOrBuilder extends
   // @@protoc_insertion_point(interface_extends:hbase.pb.LockRequest)
   org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
@@ -4898,70 +5006,2193 @@ public final class LockServiceProtos {
 
   }
 
+  public interface WaitingProcedureOrBuilder extends
+  // @@protoc_insertion_point(interface_extends:hbase.pb.WaitingProcedure)
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ * required .hbase.pb.LockType lock_type = 1;
+ */
+boolean hasLockType();
+/**
+ * required .hbase.pb.LockType lock_type = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.LockType 
getLockType();
+
+/**
+ * required 

[31/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index cc40536..d466e59 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -5778,6 +5778,1284 @@ public final class QuotaProtos {
 // @@protoc_insertion_point(class_scope:hbase.pb.SpaceLimitRequest)
   }
 
+  public interface SpaceQuotaStatusOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// optional .hbase.pb.SpaceViolationPolicy policy = 1;
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+boolean hasPolicy();
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy 
getPolicy();
+
+// optional bool in_violation = 2;
+/**
+ * optional bool in_violation = 2;
+ */
+boolean hasInViolation();
+/**
+ * optional bool in_violation = 2;
+ */
+boolean getInViolation();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.SpaceQuotaStatus}
+   *
+   * 
+   * Represents the state of a quota on a table. Either the quota is not in 
violation
+   * or it is in violatino there is a violation policy which should be in 
effect.
+   * 
+   */
+  public static final class SpaceQuotaStatus extends
+  com.google.protobuf.GeneratedMessage
+  implements SpaceQuotaStatusOrBuilder {
+// Use SpaceQuotaStatus.newBuilder() to construct.
+private SpaceQuotaStatus(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private SpaceQuotaStatus(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SpaceQuotaStatus defaultInstance;
+public static SpaceQuotaStatus getDefaultInstance() {
+  return defaultInstance;
+}
+
+public SpaceQuotaStatus getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private SpaceQuotaStatus(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 8: {
+  int rawValue = input.readEnum();
+  
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy 
value = 
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy.valueOf(rawValue);
+  if (value == null) {
+unknownFields.mergeVarintField(1, rawValue);
+  } else {
+bitField0_ |= 0x0001;
+policy_ = value;
+  }
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  inViolation_ = input.readBool();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.internal_static_hbase_pb_SpaceQuotaStatus_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 

[17/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/23194dca/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 01ba8f6..e3c6bfd 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -239,12 +239,20 @@ public final class QuotaProtos {
  * THROTTLE = 1;
  */
 THROTTLE(1),
+/**
+ * SPACE = 2;
+ */
+SPACE(2),
 ;
 
 /**
  * THROTTLE = 1;
  */
 public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
 
 
 public final int getNumber() {
@@ -262,6 +270,7 @@ public final class QuotaProtos {
 public static QuotaType forNumber(int value) {
   switch (value) {
 case 1: return THROTTLE;
+case 2: return SPACE;
 default: return null;
   }
 }
@@ -311,6 +320,150 @@ public final class QuotaProtos {
 // @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
   }
 
+  /**
+   * 
+   * Defines what action should be taken when the SpaceQuota is violated
+   * 
+   *
+   * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+   */
+  public enum SpaceViolationPolicy
+  implements 
org.apache.hadoop.hbase.shaded.com.google.protobuf.ProtocolMessageEnum {
+/**
+ * 
+ * Disable the table(s)
+ * 
+ *
+ * DISABLE = 1;
+ */
+DISABLE(1),
+/**
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+NO_WRITES_COMPACTIONS(2),
+/**
+ * 
+ * No writes or bulk-loads
+ * 
+ *
+ * NO_WRITES = 3;
+ */
+NO_WRITES(3),
+/**
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ *
+ * NO_INSERTS = 4;
+ */
+NO_INSERTS(4),
+;
+
+/**
+ * 
+ * Disable the table(s)
+ * 
+ *
+ * DISABLE = 1;
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * 
+ * No writes or bulk-loads
+ * 
+ *
+ * NO_WRITES = 3;
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ *
+ * NO_INSERTS = 4;
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() {
+  return value;
+}
+
+/**
+ * @deprecated Use {@link #forNumber(int)} instead.
+ */
+@java.lang.Deprecated
+public static SpaceViolationPolicy valueOf(int value) {
+  return forNumber(value);
+}
+
+public static SpaceViolationPolicy forNumber(int value) {
+  switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+  }
+}
+
+public static 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+  return internalValueMap;
+}
+private static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap<
+SpaceViolationPolicy> internalValueMap =
+  new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap()
 {
+public SpaceViolationPolicy findValueByNumber(int number) {
+  return SpaceViolationPolicy.forNumber(number);
+}
+  };
+
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+  return getDescriptor().getValues().get(ordinal());
+}
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+  return getDescriptor();
+}
+public static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
 desc) {
+  if (desc.getType() != getDescriptor()) {
+throw new 

[41/50] [abbrv] hbase git commit: HBASE-17568 Better handle stale/missing region size reports

[elserj]

HBASE-17568 Better handle stale/missing region size reports

* Expire region reports in the master after a timeout.
* Move regions in violation out of violation when insufficient
region size reports are observed.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d9c22cc2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d9c22cc2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d9c22cc2

Branch: refs/heads/HBASE-16961
Commit: d9c22cc28f020c80fb494bec97b51dd1cb49fe1b
Parents: 986a642
Author: Josh Elser 
Authored: Fri Feb 3 16:33:47 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:32:00 2017 -0400

--
 .../hadoop/hbase/master/MasterRpcServices.java  |   4 +-
 .../hadoop/hbase/quotas/MasterQuotaManager.java |  86 ++-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |  53 -
 .../hbase/quotas/TestMasterQuotaManager.java|  48 +++-
 .../TestQuotaObserverChoreRegionReports.java| 233 +++
 5 files changed, 412 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d9c22cc2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index d807b24..ddc58bb 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -150,6 +150,7 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Updat
 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.zookeeper.KeeperException;
@@ -1942,8 +1943,9 @@ public class MasterRpcServices extends RSRpcServices
 return RegionSpaceUseReportResponse.newBuilder().build();
   }
   MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+  final long now = EnvironmentEdgeManager.currentTime();
   for (RegionSpaceUse report : request.getSpaceUseList()) {
-quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize());
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize(), now);
   }
   return RegionSpaceUseReportResponse.newBuilder().build();
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/d9c22cc2/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index cb614ea..0622dba 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -22,9 +22,12 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -47,6 +50,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.ThrottleRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Master Quota Manager.
  * It is responsible for initialize the quota table on the first-run and
@@ -68,7 +73,7 @@ public class MasterQuotaManager implements 
RegionStateListener {
   private NamedLock userLocks;
   private boolean enabled = false;
   private NamespaceAuditor namespaceQuotaManager;
-  private ConcurrentHashMap regionSizes;
+  private ConcurrentHashMap 
regionSizes;
 
   public MasterQuotaManager(final MasterServices masterServices) {
 this.masterServices = masterServices;
@@ -531,21 +536,88 @@ public class 

[27/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
new file mode 100644
index 000..ec8f1bf
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class BaseViolationPolicyEnforcement {
+
+  static final Append APPEND = new Append(Bytes.toBytes("foo"));
+  static final Delete DELETE = new Delete(Bytes.toBytes("foo"));
+  static final Increment INCREMENT = new Increment(Bytes.toBytes("foo"));
+  static final Put PUT = new Put(Bytes.toBytes("foo"));
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
new file mode 100644
index 000..abe1b9d
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestBulkLoadCheckingViolationPolicyEnforcement {
+
+  FileSystem fs;
+  RegionServerServices rss;
+  TableName tableName;
+  SpaceViolationPolicyEnforcement policy;
+
+  @Before
+  public void setup() {
+fs = mock(FileSystem.class);
+rss = mock(RegionServerServices.class);
+tableName = TableName.valueOf("foo");
+policy = new BulkLoadVerifyingViolationPolicyEnforcement();
+  }
+
+  @Test
+  public void testFilesUnderLimit() throws Exception {
+final List paths = new ArrayList<>();
+final List 

[39/50] [abbrv] hbase git commit: HBASE-17516 Correctly handle case where table and NS quotas both apply

[elserj]

HBASE-17516 Correctly handle case where table and NS quotas both apply

The logic surrounding when a table and namespace quota both apply
to a table was incorrect, leading to a case where a table quota
violation which should have fired did not because of the less-strict
namespace quota.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/359e36c4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/359e36c4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/359e36c4

Branch: refs/heads/HBASE-16961
Commit: 359e36c43d5d18f5a4cd010a5ae7ce55379e6d2a
Parents: 609333c
Author: Josh Elser 
Authored: Wed Feb 22 18:32:55 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:32:00 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaObserverChore.java | 10 ++-
 .../TestQuotaObserverChoreWithMiniCluster.java  | 66 
 .../hbase/quotas/TestQuotaStatusRPCs.java   | 21 ++-
 .../hadoop/hbase/quotas/TestSpaceQuotas.java| 32 +-
 4 files changed, 97 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/359e36c4/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 973ac8c..b9f4592 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -287,7 +287,8 @@ public class QuotaObserverChore extends ScheduledChore {
   // We want to have a policy of "NONE", moving out of violation
   if (!targetStatus.isInViolation()) {
 for (TableName tableInNS : tablesByNamespace.get(namespace)) {
-  if 
(!tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
 {
+  // If there is a quota on this table in violation
+  if 
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
 {
 // Table-level quota violation policy is being applied here.
 if (LOG.isTraceEnabled()) {
   LOG.trace("Not activating Namespace violation policy because a 
Table violation"
@@ -298,16 +299,21 @@ public class QuotaObserverChore extends ScheduledChore {
 this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
   }
 }
+  // We want to move into violation at the NS level
   } else {
 // Moving tables in the namespace into violation or to a different 
violation policy
 for (TableName tableInNS : tablesByNamespace.get(namespace)) {
-  if 
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
 {
+  final SpaceQuotaSnapshot tableQuotaSnapshot =
+tableSnapshotStore.getCurrentState(tableInNS);
+  final boolean hasTableQuota = QuotaSnapshotStore.NO_QUOTA != 
tableQuotaSnapshot;
+  if (hasTableQuota && 
tableQuotaSnapshot.getQuotaStatus().isInViolation()) {
 // Table-level quota violation policy is being applied here.
 if (LOG.isTraceEnabled()) {
   LOG.trace("Not activating Namespace violation policy because a 
Table violation"
   + " policy is already in effect for " + tableInNS);
 }
   } else {
+// No table quota present or a table quota present that is not in 
violation
 LOG.info(tableInNS + " moving into violation of namespace space 
quota with policy " + targetStatus.getPolicy());
 this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/359e36c4/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index 943c898..63198a8 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -193,40 +193,42 @@ public class TestQuotaObserverChoreWithMiniCluster {
 
 helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
 admin.flush(tn1);
-Map violatedQuotas = 

[20/50] [abbrv] hbase git commit: HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota table (Josh Elser)

[elserj]

HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota 
table (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e0c0e950
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e0c0e950
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e0c0e950

Branch: refs/heads/HBASE-16961
Commit: e0c0e950a72dc77e1071e5d6a32816679a3ebd0e
Parents: 3a5087e
Author: tedyu 
Authored: Sat Dec 3 14:30:48 2016 -0800
Committer: Josh Elser 
Committed: Tue Apr 25 18:20:14 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  13 +-
 .../hadoop/hbase/quotas/MasterQuotaManager.java |  30 +
 .../hadoop/hbase/quotas/TestQuotaAdmin.java | 125 ++-
 3 files changed, 165 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e0c0e950/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index c44090f..8ef4f08 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -53,7 +53,9 @@ import org.apache.hadoop.hbase.util.Strings;
  * 
  * ROW-KEY  FAM/QUALDATA
  *   n.namespace q:s global-quotas
+ *   n.namespace u:dusize in bytes
  *   t.table q:s global-quotas
+ *   t.table u:dusize in bytes
  *   u.user  q:s global-quotas
  *   u.user  q:s.table table-quotas
  *   u.user  q:s.ns:   namespace-quotas
@@ -72,6 +74,7 @@ public class QuotaTableUtil {
   protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
   protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
   protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX = 
Bytes.toBytes("s.");
+  protected static final byte[] QUOTA_QUALIFIER_DISKUSAGE = 
Bytes.toBytes("du");
   protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = 
Bytes.toBytes("u.");
   protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = 
Bytes.toBytes("t.");
   protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = 
Bytes.toBytes("n.");
@@ -330,11 +333,16 @@ public class QuotaTableUtil {
*  Quotas protobuf helpers
*/
   protected static Quotas quotasFromData(final byte[] data) throws IOException 
{
+return quotasFromData(data, 0, data.length);
+  }
+
+  protected static Quotas quotasFromData(
+  final byte[] data, int offset, int length) throws IOException {
 int magicLen = ProtobufMagic.lengthOfPBMagic();
-if (!ProtobufMagic.isPBMagicPrefix(data, 0, magicLen)) {
+if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) {
   throw new IOException("Missing pb magic prefix");
 }
-return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen, 
data.length - magicLen));
+return Quotas.parseFrom(new ByteArrayInputStream(data, offset + magicLen, 
length - magicLen));
   }
 
   protected static byte[] quotasToData(final Quotas data) throws IOException {
@@ -348,6 +356,7 @@ public class QuotaTableUtil {
 boolean hasSettings = false;
 hasSettings |= quotas.hasThrottle();
 hasSettings |= quotas.hasBypassGlobals();
+hasSettings |= quotas.hasSpace();
 return !hasSettings;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e0c0e950/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 5dab2e3..1469268 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
 import 

[35/50] [abbrv] hbase git commit: HBASE-17478 Avoid reporting FS use when quotas are disabled

[elserj]

HBASE-17478 Avoid reporting FS use when quotas are disabled

Also, gracefully produce responses when quotas are disabled.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7146756
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7146756
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7146756

Branch: refs/heads/HBASE-16961
Commit: a7146756a9c1ef2494cc7f5bb1f8c5484ef34f87
Parents: 3a35c42
Author: Josh Elser 
Authored: Tue Jan 17 14:41:45 2017 -0500
Committer: Josh Elser 
Committed: Tue Apr 25 18:24:20 2017 -0400

--
 .../hadoop/hbase/master/MasterRpcServices.java  |  4 +++
 .../hadoop/hbase/quotas/MasterQuotaManager.java | 13 +--
 .../hbase/regionserver/HRegionServer.java   |  5 ++-
 .../hbase/quotas/TestMasterQuotaManager.java| 37 
 4 files changed, 56 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a7146756/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 4ee8f25..0a9d6d0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -59,6 +59,7 @@ import 
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.LockInfo;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaUtil;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -1911,6 +1912,9 @@ public class MasterRpcServices extends RSRpcServices
   RegionSpaceUseReportRequest request) throws ServiceException {
 try {
   master.checkInitialized();
+  if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) {
+return RegionSpaceUseReportResponse.newBuilder().build();
+  }
   MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
   for (RegionSpaceUse report : request.getSpaceUseList()) {
 quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize());

http://git-wip-us.apache.org/repos/asf/hbase/blob/a7146756/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index a5832f9..cb614ea 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.quotas;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -58,6 +59,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
 @InterfaceStability.Evolving
 public class MasterQuotaManager implements RegionStateListener {
   private static final Log LOG = LogFactory.getLog(MasterQuotaManager.class);
+  private static final Map EMPTY_MAP = 
Collections.unmodifiableMap(
+  new HashMap<>());
 
   private final MasterServices masterServices;
   private NamedLock namespaceLocks;
@@ -529,13 +532,19 @@ public class MasterQuotaManager implements 
RegionStateListener {
   }
 
   public void addRegionSize(HRegionInfo hri, long size) {
-// TODO Make proper API
+if (null == regionSizes) {
+  return;
+}
+// TODO Make proper API?
 // TODO Prevent from growing indefinitely
 regionSizes.put(hri, size);
   }
 
   public Map snapshotRegionSizes() {
-// TODO Make proper API
+if (null == regionSizes) {
+  return EMPTY_MAP;
+}
+// TODO Make proper API?
 return new HashMap<>(regionSizes);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/a7146756/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 

[12/50] [abbrv] hbase git commit: HBASE-15143 Procedure v2 - Web UI displaying queues

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-protocol-shaded/src/main/protobuf/LockService.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/LockService.proto 
b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
index 0df7f2e..1898e68 100644
--- a/hbase-protocol-shaded/src/main/protobuf/LockService.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/LockService.proto
@@ -25,6 +25,7 @@ option java_generate_equals_and_hash = true;
 option optimize_for = SPEED;
 
 import "HBase.proto";
+import "Procedure.proto";
 
 enum LockType {
   EXCLUSIVE = 1;
@@ -70,6 +71,27 @@ message LockProcedureData {
   optional bool is_master_lock = 6 [default = false];
 }
 
+enum ResourceType {
+  RESOURCE_TYPE_SERVER = 1;
+  RESOURCE_TYPE_NAMESPACE = 2;
+  RESOURCE_TYPE_TABLE = 3;
+  RESOURCE_TYPE_REGION = 4;
+}
+
+message WaitingProcedure {
+  required LockType lock_type = 1;
+  required Procedure procedure = 2;
+}
+
+message LockInfo {
+  required ResourceType resource_type = 1;
+  optional string resource_name = 2;
+  required LockType lock_type = 3;
+  optional Procedure exclusive_lock_owner_procedure = 4;
+  optional int32 shared_lock_count = 5;
+  repeated WaitingProcedure waitingProcedures = 6;
+}
+
 service LockService {
   /** Acquire lock on namespace/table/region */
   rpc RequestLock(LockRequest) returns(LockResponse);

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-protocol-shaded/src/main/protobuf/Master.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto 
b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index d7d51e2..0c3da02 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -30,6 +30,7 @@ import "HBase.proto";
 import "Client.proto";
 import "ClusterStatus.proto";
 import "ErrorHandling.proto";
+import "LockService.proto";
 import "Procedure.proto";
 import "Quota.proto";
 import "Replication.proto";
@@ -534,6 +535,13 @@ message ListProceduresResponse {
   repeated Procedure procedure = 1;
 }
 
+message ListLocksRequest {
+}
+
+message ListLocksResponse {
+  repeated LockInfo lock = 1;
+}
+
 message SetQuotaRequest {
   optional string user_name = 1;
   optional string user_group = 2;
@@ -888,6 +896,9 @@ service MasterService {
   rpc ListProcedures(ListProceduresRequest)
 returns(ListProceduresResponse);
 
+  rpc ListLocks(ListLocksRequest)
+returns(ListLocksResponse);
+
   /** Add a replication peer */
   rpc AddReplicationPeer(AddReplicationPeerRequest)
 returns(AddReplicationPeerResponse);

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 36d5112..e1a47c5 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -125,7 +125,7 @@ AssignmentManager assignmentManager = 
master.getAssignmentManager();
 
 Home
 Table Details
-Procedures
+Procedures  Locks
 Local Logs
 Log Level
 Debug Dump

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index aab852c..ad8aa14 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.master.RegionPlan;
 import org.apache.hadoop.hbase.master.locking.LockProcedure;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
 import org.apache.hadoop.hbase.net.Address;
+import org.apache.hadoop.hbase.procedure2.LockInfo;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -983,6 +984,24 @@ public interface MasterObserver extends Coprocessor {
   List procInfoList) throws IOException {}
 
   /**
+   * Called before a listLocks request has been processed.
+  

[04/50] [abbrv] hbase git commit: HBASE-17944 - Removed unused JDK version parsing from ClassSize.

[elserj]

HBASE-17944 - Removed unused JDK version parsing from ClassSize.

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a3b6f4ad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a3b6f4ad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a3b6f4ad

Branch: refs/heads/HBASE-16961
Commit: a3b6f4addc7ec90cbebe681e75e4e60f3e6940a5
Parents: 68e48c4
Author: Colm O hEigeartaigh 
Authored: Fri Apr 21 09:16:01 2017 +0100
Committer: Sean Busbey 
Committed: Fri Apr 21 09:16:23 2017 -0500

--
 .../java/org/apache/hadoop/hbase/util/ClassSize.java  | 14 --
 1 file changed, 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a3b6f4ad/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
index e1690c0..e064cc0 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
@@ -127,20 +127,6 @@ public class ClassSize {
 
   public static final int STORE_SERVICES;
 
-  /* Are we running on jdk7? */
-  private static final boolean JDK7;
-  static {
-final String version = System.getProperty("java.version");
-// Verify String looks like this: 1.6.0_29
-if (version == null || !version.matches("\\d\\.\\d\\..*")) {
-  throw new RuntimeException("Unexpected version format: " + version);
-}
-// Convert char to int
-int major = (int)(version.charAt(0) - '0');
-int minor = (int)(version.charAt(2) - '0');
-JDK7 = major == 1 && minor == 7;
-  }
-
   /**
* MemoryLayout abstracts details about the JVM object layout. Default 
implementation is used in
* case Unsafe is not available.

[08/50] [abbrv] hbase git commit: HBASE-17514 emit a warning if thrift1 proxy user is configured but hbase.regionserver.thrift.http is not

[elserj]

HBASE-17514 emit a warning if thrift1 proxy user is configured but 
hbase.regionserver.thrift.http is not

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a1aff44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a1aff44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a1aff44

Branch: refs/heads/HBASE-16961
Commit: 9a1aff447e908c9de351a4f45b869b016ad7821b
Parents: 435104a
Author: lv zehui 
Authored: Sat Apr 22 21:20:00 2017 +0800
Committer: Sean Busbey 
Committed: Mon Apr 24 11:33:27 2017 -0500

--
 .../java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9a1aff44/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
--
diff --git 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
index 0829188..6a074fd 100644
--- 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
+++ 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
@@ -333,6 +333,11 @@ public class ThriftServerRunner implements Runnable {
 this.realUser = userProvider.getCurrent().getUGI();
 qop = conf.get(THRIFT_QOP_KEY);
 doAsEnabled = conf.getBoolean(THRIFT_SUPPORT_PROXYUSER, false);
+if (doAsEnabled) {
+  if (!conf.getBoolean(USE_HTTP_CONF_KEY, false)) {
+LOG.warn("Fail to enable the doAs feature. 
hbase.regionserver.thrift.http is not configured ");
+  }
+}
 if (qop != null) {
   if (!qop.equals("auth") && !qop.equals("auth-int")
   && !qop.equals("auth-conf")) {

[11/50] [abbrv] hbase git commit: HBASE-17947 Location of Examples.proto is wrong in comment of RowCountEndPoint.java

[elserj]

HBASE-17947 Location of Examples.proto is wrong in comment of 
RowCountEndPoint.java

Signed-off-by: tedyu 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1367519c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1367519c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1367519c

Branch: refs/heads/HBASE-16961
Commit: 1367519cd0545c2854108cffab03ae7c79b6ef2c
Parents: 72fac37
Author: Xiang Li 
Authored: Fri Apr 21 19:17:49 2017 +0800
Committer: tedyu 
Committed: Tue Apr 25 01:48:57 2017 -0700

--
 .../apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1367519c/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.java
--
diff --git 
a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.java
 
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.java
index 36d8488..598008b 100644
--- 
a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.java
+++ 
b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/RowCountEndpoint.java
@@ -45,7 +45,7 @@ import com.google.protobuf.Service;
  *
  * 
  * For the protocol buffer definition of the RowCountService, see the source 
file located under
- * hbase-server/src/main/protobuf/Examples.proto.
+ * hbase-examples/src/main/protobuf/Examples.proto.
  * 
  */
 public class RowCountEndpoint extends ExampleProtos.RowCountService

[01/50] [abbrv] hbase git commit: HBASE-17941 CellArrayMap#getCell may throw IndexOutOfBoundsException [Forced Update!]

[elserj]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-16961 2524f8cd2 -> cb08814a4 (forced update)


HBASE-17941 CellArrayMap#getCell may throw IndexOutOfBoundsException

Signed-off-by: Chia-Ping Tsai 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/33dadc1a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/33dadc1a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/33dadc1a

Branch: refs/heads/HBASE-16961
Commit: 33dadc1a941a536742799a46444c67a1ed66d124
Parents: ea3a27b
Author: s9514171 
Authored: Thu Apr 20 14:54:52 2017 +0800
Committer: Chia-Ping Tsai 
Committed: Fri Apr 21 11:35:39 2017 +0800

--
 .../java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/33dadc1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java
index 605fea2..898e469 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayMap.java
@@ -48,7 +48,7 @@ public class CellArrayMap extends CellFlatMap {
 
   @Override
   protected Cell getCell(int i) {
-if( (i < minCellIdx) && (i >= maxCellIdx) ) return null;
+if( (i < minCellIdx) || (i >= maxCellIdx) ) return null;
 return block[i];
   }
 }

[03/50] [abbrv] hbase git commit: HBASE-17946 Shell command compact_rs don't work (Guangxu Cheng)

[elserj]

HBASE-17946 Shell command compact_rs don't work (Guangxu Cheng)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/68e48c45
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/68e48c45
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/68e48c45

Branch: refs/heads/HBASE-16961
Commit: 68e48c456dc018775df792507087bf275bf3304f
Parents: d39f40e
Author: tedyu 
Authored: Fri Apr 21 06:54:44 2017 -0700
Committer: tedyu 
Committed: Fri Apr 21 06:54:44 2017 -0700

--
 hbase-shell/src/main/ruby/shell/commands/compact_rs.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/68e48c45/hbase-shell/src/main/ruby/shell/commands/compact_rs.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/compact_rs.rb 
b/hbase-shell/src/main/ruby/shell/commands/compact_rs.rb
index 588b6fe..5f02944 100644
--- a/hbase-shell/src/main/ruby/shell/commands/compact_rs.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/compact_rs.rb
@@ -34,7 +34,7 @@ module Shell
   end
 
   def command(regionserver, major = false)
-admin.compactRegionserver(regionserver, major)
+admin.compact_regionserver(regionserver, major)
   end
 end
   end

[13/50] [abbrv] hbase git commit: HBASE-15143 Procedure v2 - Web UI displaying queues

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index 8ff19b2..e4ce4cb 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -62144,6 +62144,1133 @@ public final class MasterProtos {
 
   }
 
+  public interface ListLocksRequestOrBuilder extends
+  // @@protoc_insertion_point(interface_extends:hbase.pb.ListLocksRequest)
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+  }
+  /**
+   * Protobuf type {@code hbase.pb.ListLocksRequest}
+   */
+  public  static final class ListLocksRequest extends
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3 
implements
+  // @@protoc_insertion_point(message_implements:hbase.pb.ListLocksRequest)
+  ListLocksRequestOrBuilder {
+// Use ListLocksRequest.newBuilder() to construct.
+private 
ListLocksRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
 builder) {
+  super(builder);
+}
+private ListLocksRequest() {
+}
+
+@java.lang.Override
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private ListLocksRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream 
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite 
extensionRegistry)
+throws 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 {
+  this();
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder 
unknownFields =
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+  }
+}
+  } catch 
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.internal_static_hbase_pb_ListLocksRequest_descriptor;
+}
+
+protected 
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.internal_static_hbase_pb_ListLocksRequest_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListLocksRequest.class,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListLocksRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+  byte isInitialized = memoizedIsInitialized;
+  if (isInitialized == 1) return true;
+  if (isInitialized == 0) return false;
+
+  memoizedIsInitialized = 1;
+  return true;
+}
+
+public void 
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream 
output)
+throws java.io.IOException {
+  unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+  int size = memoizedSize;
+  if (size != -1) return size;
+
+  size = 0;
+  size += unknownFields.getSerializedSize();
+  memoizedSize = size;
+  return size;
+}
+
+private static final long serialVersionUID = 0L;
+@java.lang.Override
+public boolean equals(final java.lang.Object obj) {
+  if (obj == this) {
+   return true;
+  }
+  if (!(obj instanceof 

[09/50] [abbrv] hbase git commit: HBASE-17933: [hbase-spark] Support Java api for bulkload

[elserj]

HBASE-17933: [hbase-spark] Support Java api for bulkload

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/49f707fb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/49f707fb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/49f707fb

Branch: refs/heads/HBASE-16961
Commit: 49f707fba7c6a9f0210f387e31d1be9f108991f8
Parents: 9a1aff4
Author: Yi Liang 
Authored: Fri Apr 21 18:10:03 2017 -0700
Committer: Sean Busbey 
Committed: Mon Apr 24 11:48:29 2017 -0500

--
 .../hbasecontext/JavaHBaseBulkLoadExample.java  | 102 ++
 .../hbase/spark/FamiliesQualifiersValues.scala  |  12 +-
 .../hadoop/hbase/spark/JavaHBaseContext.scala   |  68 ++-
 .../hbase/spark/TestJavaHBaseContext.java   | 201 ++-
 4 files changed, 371 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/49f707fb/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/example/hbasecontext/JavaHBaseBulkLoadExample.java
--
diff --git 
a/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/example/hbasecontext/JavaHBaseBulkLoadExample.java
 
b/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/example/hbasecontext/JavaHBaseBulkLoadExample.java
new file mode 100644
index 000..040546d
--- /dev/null
+++ 
b/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/example/hbasecontext/JavaHBaseBulkLoadExample.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.spark.example.hbasecontext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.spark.FamilyHFileWriteOptions;
+import org.apache.hadoop.hbase.spark.JavaHBaseContext;
+import org.apache.hadoop.hbase.spark.KeyFamilyQualifier;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.function.Function;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Run this example using command below:
+ *
+ *  SPARK_HOME/bin/spark-submit --master local[2] --class 
org.apache.hadoop.hbase.spark.example.hbasecontext.JavaHBaseBulkLoadExample
+ *  path/to/hbase-spark.jar {path/to/output/HFiles}
+ *
+ * This example will output put hfiles in {path/to/output/HFiles}, and user 
can run
+ * 'hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles' to load the 
HFiles into table to verify this example.
+ */
+final public class JavaHBaseBulkLoadExample {
+  private JavaHBaseBulkLoadExample() {}
+
+  public static void main(String[] args) {
+if (args.length < 1) {
+  System.out.println("JavaHBaseBulkLoadExample  " + "{outputPath}");
+  return;
+}
+
+String tableName = "bulkload-table-test";
+String columnFamily1 = "f1";
+String columnFamily2 = "f2";
+
+SparkConf sparkConf = new SparkConf().setAppName("JavaHBaseBulkLoadExample 
" + tableName);
+JavaSparkContext jsc = new JavaSparkContext(sparkConf);
+
+try {
+  List list= new ArrayList();
+  // row1
+  list.add("1," + columnFamily1 + ",b,1");
+  // row3
+  list.add("3," + columnFamily1 + ",a,2");
+  list.add("3," + columnFamily1 + ",b,1");
+  list.add("3," + columnFamily2 + ",a,1");
+  /* row2 */
+  list.add("2," + columnFamily2 + ",a,3");
+  list.add("2," + columnFamily2 + ",b,3");
+
+  JavaRDD rdd = jsc.parallelize(list);
+
+  Configuration conf = HBaseConfiguration.create();
+  JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
+
+
+
+  hbaseContext.bulkLoad(rdd, TableName.valueOf(tableName),new 
BulkLoadFunction(), 

[45/50] [abbrv] hbase git commit: HBASE-17447 Implement a MasterObserver for automatically deleting space quotas

[elserj]

HBASE-17447 Implement a MasterObserver for automatically deleting space quotas

When a table or namespace is deleted, it would be nice to automatically
delete the quota on said table/NS. It's possible that not all people
would want this functionality so we can leave it up to the user to
configure this Observer.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2524f8cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2524f8cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2524f8cd

Branch: refs/heads/HBASE-16961
Commit: 2524f8cd26422ff7a44cca2aa6c913f713c1d530
Parents: abe1c06
Author: Josh Elser 
Authored: Thu Mar 16 18:54:01 2017 -0400
Committer: Josh Elser 
Committed: Mon Apr 17 15:47:49 2017 -0400

--
 .../hbase/quotas/MasterSpaceQuotaObserver.java  |  85 ++
 .../quotas/TestMasterSpaceQuotaObserver.java| 169 +++
 src/main/asciidoc/_chapters/ops_mgt.adoc|  17 ++
 3 files changed, 271 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2524f8cd/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
new file mode 100644
index 000..a3abf32
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+
+/**
+ * An observer to automatically delete space quotas when a table/namespace
+ * are deleted.
+ */
+@InterfaceAudience.Private
+public class MasterSpaceQuotaObserver implements MasterObserver {
+  private CoprocessorEnvironment cpEnv;
+  private Configuration conf;
+  private boolean quotasEnabled = false;
+
+  @Override
+  public void start(CoprocessorEnvironment ctx) throws IOException {
+this.cpEnv = ctx;
+this.conf = cpEnv.getConfiguration();
+this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf);
+  }
+
+  @Override
+  public void postDeleteTable(
+  ObserverContext ctx, TableName tableName) 
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+  return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+if (null != quotas && quotas.hasSpace()) {
+  QuotaSettings settings = 
QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+  try (Admin admin = conn.getAdmin()) {
+admin.setQuota(settings);
+  }
+}
+  }
+
+  @Override
+  public void postDeleteNamespace(
+  ObserverContext ctx, String namespace) 
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+  return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(), 
namespace);
+if (null != quotas && quotas.hasSpace()) {
+  

[23/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/990062a9/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 01ba8f6..e3c6bfd 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -239,12 +239,20 @@ public final class QuotaProtos {
  * THROTTLE = 1;
  */
 THROTTLE(1),
+/**
+ * SPACE = 2;
+ */
+SPACE(2),
 ;
 
 /**
  * THROTTLE = 1;
  */
 public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
 
 
 public final int getNumber() {
@@ -262,6 +270,7 @@ public final class QuotaProtos {
 public static QuotaType forNumber(int value) {
   switch (value) {
 case 1: return THROTTLE;
+case 2: return SPACE;
 default: return null;
   }
 }
@@ -311,6 +320,150 @@ public final class QuotaProtos {
 // @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
   }
 
+  /**
+   * 
+   * Defines what action should be taken when the SpaceQuota is violated
+   * 
+   *
+   * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+   */
+  public enum SpaceViolationPolicy
+  implements 
org.apache.hadoop.hbase.shaded.com.google.protobuf.ProtocolMessageEnum {
+/**
+ * 
+ * Disable the table(s)
+ * 
+ *
+ * DISABLE = 1;
+ */
+DISABLE(1),
+/**
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+NO_WRITES_COMPACTIONS(2),
+/**
+ * 
+ * No writes or bulk-loads
+ * 
+ *
+ * NO_WRITES = 3;
+ */
+NO_WRITES(3),
+/**
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ *
+ * NO_INSERTS = 4;
+ */
+NO_INSERTS(4),
+;
+
+/**
+ * 
+ * Disable the table(s)
+ * 
+ *
+ * DISABLE = 1;
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * 
+ * No writes or bulk-loads
+ * 
+ *
+ * NO_WRITES = 3;
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ *
+ * NO_INSERTS = 4;
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() {
+  return value;
+}
+
+/**
+ * @deprecated Use {@link #forNumber(int)} instead.
+ */
+@java.lang.Deprecated
+public static SpaceViolationPolicy valueOf(int value) {
+  return forNumber(value);
+}
+
+public static SpaceViolationPolicy forNumber(int value) {
+  switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+  }
+}
+
+public static 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+  return internalValueMap;
+}
+private static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap<
+SpaceViolationPolicy> internalValueMap =
+  new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap()
 {
+public SpaceViolationPolicy findValueByNumber(int number) {
+  return SpaceViolationPolicy.forNumber(number);
+}
+  };
+
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+  return getDescriptor().getValues().get(ordinal());
+}
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+  return getDescriptor();
+}
+public static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
 desc) {
+  if (desc.getType() != getDescriptor()) {
+throw new 

[41/50] [abbrv] hbase git commit: HBASE-17428 Implement informational RPCs for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/095fabf1/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index e90c934..c70b736 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10429,7 +10429,7 @@ public final class RegionServerStatusProtos {
 return memoizedHashCode;
   }
   int hash = 41;
-  hash = (19 * hash) + getDescriptorForType().hashCode();
+  hash = (19 * hash) + getDescriptor().hashCode();
   if (hasRegion()) {
 hash = (37 * hash) + REGION_FIELD_NUMBER;
 hash = (53 * hash) + getRegion().hashCode();
@@ -10824,7 +10824,7 @@ public final class RegionServerStatusProtos {
* optional .hbase.pb.RegionInfo region = 1;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo, 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo, 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
 
   getRegionFieldBuilder() {
 if (regionBuilder_ == null) {
   regionBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -10940,7 +10940,7 @@ public final class RegionServerStatusProtos {
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
  */
-
java.util.List
+
java.util.List
 
 getSpaceUseList();
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
@@ -10953,7 +10953,7 @@ public final class RegionServerStatusProtos {
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
  */
-java.util.List
+java.util.List
 
 getSpaceUseOrBuilderList();
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
@@ -11056,7 +11056,7 @@ public final class RegionServerStatusProtos {
 /**
  * repeated .hbase.pb.RegionSpaceUse space_use = 1;
  */
-public java.util.List
+public java.util.List
 
 getSpaceUseOrBuilderList() {
   return spaceUse_;
 }
@@ -11142,7 +11142,7 @@ public final class RegionServerStatusProtos {
 return memoizedHashCode;
   }
   int hash = 41;
-  hash = (19 * hash) + getDescriptorForType().hashCode();
+  hash = (19 * hash) + getDescriptor().hashCode();
   if (getSpaceUseCount() > 0) {
 hash = (37 * hash) + SPACE_USE_FIELD_NUMBER;
 hash = (53 * hash) + getSpaceUseList().hashCode();
@@ -11368,7 +11368,7 @@ public final class RegionServerStatusProtos {
   spaceUseBuilder_ = null;
   spaceUse_ = other.spaceUse_;
   bitField0_ = (bitField0_ & ~0x0001);
-  spaceUseBuilder_ =
+  spaceUseBuilder_ = 
 
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
 ?
getSpaceUseFieldBuilder() : null;
 } else {
@@ -11604,7 +11604,7 @@ public final class RegionServerStatusProtos {
   /**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-  public java.util.List
+  public java.util.List
 
getSpaceUseOrBuilderList() {
 if (spaceUseBuilder_ != null) {
   return spaceUseBuilder_.getMessageOrBuilderList();
@@ -11630,12 +11630,12 @@ public final class RegionServerStatusProtos {
   /**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-  public 
java.util.List
+  public 
java.util.List
 
getSpaceUseBuilderList() {
 return getSpaceUseFieldBuilder().getBuilderList();
   }
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.RepeatedFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseOrBuilder>
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse,
 

[48/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-protocol-shaded/src/main/protobuf/Master.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto 
b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 3318a39..beb8f02 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -930,7 +930,11 @@ service MasterService {
   rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
 returns(RemoveDrainFromRegionServersResponse);
 
-  /** Fetches the Master's view of space quotas */
+  /** Fetches the Master's view of space utilization */
   rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
 returns(GetSpaceQuotaRegionSizesResponse);
+
+  /** Fetches the Master's view of quotas */
+  rpc GetQuotaStates(GetQuotaStatesRequest)
+returns(GetQuotaStatesResponse);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto 
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 2d7e5f5..1a6d5ed 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
   message RegionSizes {
 optional TableName table_name = 1;
 optional uint64 size = 2;
+
   }
   repeated RegionSizes sizes = 1;
 }
@@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
   }
   repeated TableViolationPolicy violation_policies = 1;
 }
+
+message GetQuotaStatesRequest {
+}
+
+message GetQuotaStatesResponse {
+  message TableQuotaSnapshot {
+optional TableName table_name = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+  }
+  message NamespaceQuotaSnapshot {
+optional string namespace = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+  }
+  repeated TableQuotaSnapshot table_snapshots = 1;
+  repeated NamespaceQuotaSnapshot ns_snapshots = 2;
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0243934..92c1461 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -908,7 +908,7 @@ public class HMaster extends HRegionServer implements 
MasterServices {
   // Create the quota snapshot notifier
   spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
   spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
-  this.quotaObserverChore = new QuotaObserverChore(this);
+  this.quotaObserverChore = new QuotaObserverChore(this, 
getMasterMetrics());
   // Start the chore to read the region FS space reports and act on them
   getChoreService().scheduleChore(quotaObserverChore);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 3d72a3e..8f796bb 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -62,7 +62,9 @@ import 
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
 import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -214,8 +216,12 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import 

[30/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
new file mode 100644
index 000..6b754b9
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import 
org.apache.hadoop.hbase.quotas.policies.BulkLoadVerifyingViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.DisableTableViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.NoInsertsViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.NoWritesCompactionsViolationPolicyEnforcement;
+import 
org.apache.hadoop.hbase.quotas.policies.NoWritesViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * A factory class for instantiating {@link SpaceViolationPolicyEnforcement} 
instances.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SpaceViolationPolicyEnforcementFactory {
+
+  private static final SpaceViolationPolicyEnforcementFactory INSTANCE =
+  new SpaceViolationPolicyEnforcementFactory();
+
+  private SpaceViolationPolicyEnforcementFactory() {}
+
+  /**
+   * Returns an instance of this factory.
+   */
+  public static SpaceViolationPolicyEnforcementFactory getInstance() {
+return INSTANCE;
+  }
+
+  /**
+   * Constructs the appropriate {@link SpaceViolationPolicyEnforcement} for 
tables that are
+   * in violation of their space quota.
+   */
+  public SpaceViolationPolicyEnforcement create(
+  RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot 
snapshot) {
+SpaceViolationPolicyEnforcement enforcement;
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (!status.isInViolation()) {
+  throw new IllegalArgumentException(tableName + " is not in violation. 
Snapshot=" + snapshot);
+}
+switch (status.getPolicy()) {
+  case DISABLE:
+enforcement = new DisableTableViolationPolicyEnforcement();
+break;
+  case NO_WRITES_COMPACTIONS:
+enforcement = new NoWritesCompactionsViolationPolicyEnforcement();
+break;
+  case NO_WRITES:
+enforcement = new NoWritesViolationPolicyEnforcement();
+break;
+  case NO_INSERTS:
+enforcement = new NoInsertsViolationPolicyEnforcement();
+break;
+  default:
+throw new IllegalArgumentException("Unhandled SpaceViolationPolicy: " 
+ status.getPolicy());
+}
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+  }
+
+  /**
+   * Creates the "default" {@link SpaceViolationPolicyEnforcement} for a table 
that isn't in
+   * violation. This is used to have uniform policy checking for tables in and 
not quotas.
+   */
+  public SpaceViolationPolicyEnforcement createWithoutViolation(
+  RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot 
snapshot) {
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (status.isInViolation()) {
+  throw new IllegalArgumentException(
+  tableName + " is in violation. Logic error. Snapshot=" + snapshot);
+}
+BulkLoadVerifyingViolationPolicyEnforcement enforcement = new 
BulkLoadVerifyingViolationPolicyEnforcement();
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java

[07/50] [abbrv] hbase git commit: HBASE-17866: Implement async setQuota/getQuota methods

[elserj]

HBASE-17866: Implement async setQuota/getQuota methods

Signed-off-by: Guanghao Zhang 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8db97603
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8db97603
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8db97603

Branch: refs/heads/HBASE-16961
Commit: 8db9760363890d4d0bfaba25ae6797d45aaf7fec
Parents: 7678855
Author: huzheng 
Authored: Fri Apr 14 14:51:38 2017 +0800
Committer: Guanghao Zhang 
Committed: Mon Apr 17 09:49:30 2017 +0800

--
 .../apache/hadoop/hbase/client/AsyncAdmin.java  |  16 ++
 .../hadoop/hbase/client/AsyncHBaseAdmin.java|  47 +
 .../hadoop/hbase/quotas/QuotaRetriever.java |  32 +--
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  32 +++
 .../hbase/client/TestAsyncQuotaAdminApi.java| 207 +++
 5 files changed, 306 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8db97603/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index ab791c2..270f28f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.regex.Pattern;
 
@@ -27,6 +28,8 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.quotas.QuotaFilter;
+import org.apache.hadoop.hbase.quotas.QuotaSettings;
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
@@ -465,4 +468,17 @@ public interface AsyncAdmin {
*  startcode. Here is an example:  
host187.example.com,60020,1289493121758
*/
   CompletableFuture move(final byte[] regionName, final byte[] 
destServerName);
+
+  /**
+   * Apply the new quota settings.
+   * @param quota the quota settings
+   */
+  CompletableFuture setQuota(final QuotaSettings quota);
+
+  /**
+   * List the quotas based on the filter.
+   * @param filter the quota settings filter
+   * @return the QuotaSetting list, which wrapped by a CompletableFuture.
+   */
+  CompletableFuture getQuota(QuotaFilter filter);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/8db97603/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index e42ee57..180cd19 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -56,6 +56,9 @@ import 
org.apache.hadoop.hbase.client.AsyncRpcRetryingCallerFactory.MasterReques
 import org.apache.hadoop.hbase.client.Scan.ReadType;
 import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.hadoop.hbase.quotas.QuotaFilter;
+import org.apache.hadoop.hbase.quotas.QuotaSettings;
+import org.apache.hadoop.hbase.quotas.QuotaTableUtil;
 import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback;
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
@@ -112,6 +115,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineReg
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
@@ -1149,6 +1154,48 @@ public class 

[35/50] [abbrv] hbase git commit: HBASE-17259 API to remove space quotas on a table/namespace

[elserj]

HBASE-17259 API to remove space quotas on a table/namespace


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0effca42
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0effca42
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0effca42

Branch: refs/heads/HBASE-16961
Commit: 0effca421562066d10d13575f5c12434e2741c63
Parents: 0d76d66
Author: Josh Elser 
Authored: Wed Jan 11 12:47:06 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:32 2017 -0400

--
 .../hbase/quotas/QuotaSettingsFactory.java  |  22 +++
 .../hadoop/hbase/quotas/QuotaTableUtil.java |   6 +-
 .../hadoop/hbase/quotas/SpaceLimitSettings.java |  44 -
 .../hbase/quotas/TestQuotaSettingsFactory.java  |  20 +++
 .../shaded/protobuf/generated/QuotaProtos.java  | 157 +++---
 .../src/main/protobuf/Quota.proto   |   1 +
 .../hbase/protobuf/generated/QuotaProtos.java   | 159 ---
 hbase-protocol/src/main/protobuf/Quota.proto|   1 +
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   9 +-
 .../hadoop/hbase/quotas/TestQuotaAdmin.java |  49 +-
 10 files changed, 423 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0effca42/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 7f1c180..184277d 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -316,6 +316,17 @@ public class QuotaSettingsFactory {
   }
 
   /**
+   * Creates a {@link QuotaSettings} object to remove the FileSystem space 
quota for the given
+   * table.
+   *
+   * @param tableName The name of the table to remove the quota for.
+   * @return A {@link QuotaSettings} object.
+   */
+  public static QuotaSettings removeTableSpaceLimit(TableName tableName) {
+return new SpaceLimitSettings(tableName, true);
+  }
+
+  /**
* Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given
* namespace to the given size in bytes. When the space usage is exceeded by 
all tables in the
* namespace, the provided {@link SpaceViolationPolicy} is enacted on all 
tables in the namespace.
@@ -329,4 +340,15 @@ public class QuotaSettingsFactory {
   final String namespace, long sizeLimit, final SpaceViolationPolicy 
violationPolicy) {
 return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
   }
+
+  /**
+   * Creates a {@link QuotaSettings} object to remove the FileSystem space 
quota for the given
+* namespace.
+   *
+   * @param namespace The namespace to remove the quota on.
+   * @return A {@link QuotaSettings} object.
+   */
+  public static QuotaSettings removeNamespaceSpaceLimit(String namespace) {
+return new SpaceLimitSettings(namespace, true);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0effca42/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 66535b2..ce4cd04 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -422,7 +422,11 @@ public class QuotaTableUtil {
 boolean hasSettings = false;
 hasSettings |= quotas.hasThrottle();
 hasSettings |= quotas.hasBypassGlobals();
-hasSettings |= quotas.hasSpace();
+// Only when there is a space quota, make sure there's actually both 
fields provided
+// Otherwise, it's a noop.
+if (quotas.hasSpace()) {
+  hasSettings |= (quotas.getSpace().hasSoftLimit() && 
quotas.getSpace().hasViolationPolicy());
+}
 return !hasSettings;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/0effca42/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
index e54882e..8ff7623 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
+++ 

[18/50] [abbrv] hbase git commit: HBASE-16998 Implement Master-side analysis of region space reports

[elserj]

HBASE-16998 Implement Master-side analysis of region space reports

Adds a new Chore to the Master that analyzes the reports that are
sent by RegionServers. The Master must then, for all tables with
quotas, determine the tables that are violating quotas and move
those tables into violation. Similarly, tables no longer violating
the quota can be moved out of violation.

The Chore is the "stateful" bit, managing which tables are and
are not in violation. Everything else is just performing
computation and informing the Chore on the updated state.

Added InterfaceAudience annotations and clean up the QuotaObserverChore
constructor. Cleaned up some javadoc and QuotaObserverChore. Reuse
the QuotaViolationStore impl objects.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bcf6da40
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bcf6da40
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bcf6da40

Branch: refs/heads/HBASE-16961
Commit: bcf6da40514f0858c8f63cb9ac379dbcd28d1058
Parents: d77ec49
Author: Josh Elser 
Authored: Tue Nov 8 18:55:12 2016 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:31 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaRetriever.java |  27 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  20 +
 .../hadoop/hbase/quotas/MasterQuotaManager.java |   1 +
 .../quotas/NamespaceQuotaViolationStore.java| 127 
 .../hadoop/hbase/quotas/QuotaObserverChore.java | 618 +++
 .../hbase/quotas/QuotaViolationStore.java   |  89 +++
 .../quotas/SpaceQuotaViolationNotifier.java |  44 ++
 .../SpaceQuotaViolationNotifierForTest.java |  50 ++
 .../hbase/quotas/TableQuotaViolationStore.java  | 127 
 .../TestNamespaceQuotaViolationStore.java   | 156 +
 .../hbase/quotas/TestQuotaObserverChore.java| 106 
 .../TestQuotaObserverChoreWithMiniCluster.java  | 596 ++
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |   4 -
 .../quotas/TestTableQuotaViolationStore.java| 151 +
 .../hbase/quotas/TestTablesWithQuotas.java  | 198 ++
 15 files changed, 2305 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bcf6da40/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
index 0f7baa5..4482693 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.LinkedList;
+import java.util.Objects;
 import java.util.Queue;
 
 import org.apache.commons.logging.Log;
@@ -54,11 +55,23 @@ public class QuotaRetriever implements Closeable, 
Iterable {
   private Connection connection;
   private Table table;
 
-  private QuotaRetriever() {
+  /**
+   * Should QutoaRetriever manage the state of the connection, or leave it be.
+   */
+  private boolean isManagedConnection = false;
+
+  QuotaRetriever() {
   }
 
   void init(final Configuration conf, final Scan scan) throws IOException {
-this.connection = ConnectionFactory.createConnection(conf);
+// Set this before creating the connection and passing it down to make sure
+// it's cleaned up if we fail to construct the Scanner.
+this.isManagedConnection = true;
+init(ConnectionFactory.createConnection(conf), scan);
+  }
+
+  void init(final Connection conn, final Scan scan) throws IOException {
+this.connection = Objects.requireNonNull(conn);
 this.table = this.connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
 try {
   scanner = table.getScanner(scan);
@@ -77,10 +90,14 @@ public class QuotaRetriever implements Closeable, 
Iterable {
   this.table.close();
   this.table = null;
 }
-if (this.connection != null) {
-  this.connection.close();
-  this.connection = null;
+// Null out the connection on close() even if we didn't explicitly close it
+// to maintain typical semantics.
+if (isManagedConnection) {
+  if (this.connection != null) {
+this.connection.close();
+  }
 }
+this.connection = null;
   }
 
   public QuotaSettings next() throws IOException {

http://git-wip-us.apache.org/repos/asf/hbase/blob/bcf6da40/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git 

[40/50] [abbrv] hbase git commit: HBASE-17568 Better handle stale/missing region size reports

[elserj]

HBASE-17568 Better handle stale/missing region size reports

* Expire region reports in the master after a timeout.
* Move regions in violation out of violation when insufficient
region size reports are observed.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/71e14a3c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/71e14a3c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/71e14a3c

Branch: refs/heads/HBASE-16961
Commit: 71e14a3cccfbe7125284a32250e4aefc0d575b30
Parents: 4db44ad
Author: Josh Elser 
Authored: Fri Feb 3 16:33:47 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:44:00 2017 -0400

--
 .../hadoop/hbase/master/MasterRpcServices.java  |   4 +-
 .../hadoop/hbase/quotas/MasterQuotaManager.java |  86 ++-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |  53 -
 .../hbase/quotas/TestMasterQuotaManager.java|  48 +++-
 .../TestQuotaObserverChoreRegionReports.java| 233 +++
 5 files changed, 412 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/71e14a3c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 42f99b2..3d72a3e 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -251,6 +251,7 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Updat
 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.zookeeper.KeeperException;
@@ -2027,8 +2028,9 @@ public class MasterRpcServices extends RSRpcServices
 return RegionSpaceUseReportResponse.newBuilder().build();
   }
   MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+  final long now = EnvironmentEdgeManager.currentTime();
   for (RegionSpaceUse report : request.getSpaceUseList()) {
-quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize());
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize(), now);
   }
   return RegionSpaceUseReportResponse.newBuilder().build();
 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/71e14a3c/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index cb614ea..0622dba 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -22,9 +22,12 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -47,6 +50,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.ThrottleRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Master Quota Manager.
  * It is responsible for initialize the quota table on the first-run and
@@ -68,7 +73,7 @@ public class MasterQuotaManager implements 
RegionStateListener {
   private NamedLock userLocks;
   private boolean enabled = false;
   private NamespaceAuditor namespaceQuotaManager;
-  private ConcurrentHashMap regionSizes;
+  private ConcurrentHashMap 
regionSizes;
 
   public MasterQuotaManager(final MasterServices masterServices) {
 this.masterServices = masterServices;
@@ -531,21 +536,88 @@ public class 

[28/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
new file mode 100644
index 000..ec8f1bf
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class BaseViolationPolicyEnforcement {
+
+  static final Append APPEND = new Append(Bytes.toBytes("foo"));
+  static final Delete DELETE = new Delete(Bytes.toBytes("foo"));
+  static final Increment INCREMENT = new Increment(Bytes.toBytes("foo"));
+  static final Put PUT = new Put(Bytes.toBytes("foo"));
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
new file mode 100644
index 000..abe1b9d
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestBulkLoadCheckingViolationPolicyEnforcement {
+
+  FileSystem fs;
+  RegionServerServices rss;
+  TableName tableName;
+  SpaceViolationPolicyEnforcement policy;
+
+  @Before
+  public void setup() {
+fs = mock(FileSystem.class);
+rss = mock(RegionServerServices.class);
+tableName = TableName.valueOf("foo");
+policy = new BulkLoadVerifyingViolationPolicyEnforcement();
+  }
+
+  @Test
+  public void testFilesUnderLimit() throws Exception {
+final List paths = new ArrayList<>();
+final List 

[22/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/990062a9/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index 05894b9..1925828 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -217,12 +217,20 @@ public final class QuotaProtos {
  * THROTTLE = 1;
  */
 THROTTLE(0, 1),
+/**
+ * SPACE = 2;
+ */
+SPACE(1, 2),
 ;
 
 /**
  * THROTTLE = 1;
  */
 public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
 
 
 public final int getNumber() { return value; }
@@ -230,6 +238,7 @@ public final class QuotaProtos {
 public static QuotaType valueOf(int value) {
   switch (value) {
 case 1: return THROTTLE;
+case 2: return SPACE;
 default: return null;
   }
 }
@@ -281,6 +290,142 @@ public final class QuotaProtos {
 // @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
   }
 
+  /**
+   * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+   *
+   * 
+   * Defines what action should be taken when the SpaceQuota is violated
+   * 
+   */
+  public enum SpaceViolationPolicy
+  implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * DISABLE = 1;
+ *
+ * 
+ * Disable the table(s)
+ * 
+ */
+DISABLE(0, 1),
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ */
+NO_WRITES_COMPACTIONS(1, 2),
+/**
+ * NO_WRITES = 3;
+ *
+ * 
+ * No writes or bulk-loads
+ * 
+ */
+NO_WRITES(2, 3),
+/**
+ * NO_INSERTS = 4;
+ *
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ */
+NO_INSERTS(3, 4),
+;
+
+/**
+ * DISABLE = 1;
+ *
+ * 
+ * Disable the table(s)
+ * 
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ * 
+ * No writes, bulk-loads, or compactions
+ * 
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * NO_WRITES = 3;
+ *
+ * 
+ * No writes or bulk-loads
+ * 
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * NO_INSERTS = 4;
+ *
+ * 
+ * No puts or bulk-loads, but deletes are allowed
+ * 
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() { return value; }
+
+public static SpaceViolationPolicy valueOf(int value) {
+  switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+  }
+}
+
+public static 
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+  return internalValueMap;
+}
+private static 
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+  new com.google.protobuf.Internal.EnumLiteMap() 
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+  return SpaceViolationPolicy.valueOf(number);
+}
+  };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+  return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+  return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+  if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+  "EnumValueDescriptor is not for this type.");
+  }
+  return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private SpaceViolationPolicy(int index, int value) {
+  this.index = index;
+  this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.SpaceViolationPolicy)
+  }
+
   public interface TimedQuotaOrBuilder
   extends com.google.protobuf.MessageOrBuilder {
 
@@ -3315,6 +3460,20 @@ public final class QuotaProtos {
 

[26/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages - addendum fixes line lengths (Josh Elser)

[elserj]

HBASE-16995 Build client Java API and client protobuf messages - addendum fixes 
line lengths (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/988a23ef
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/988a23ef
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/988a23ef

Branch: refs/heads/HBASE-16961
Commit: 988a23eff38e460ec383544930d48147d4771d4c
Parents: eaeef44
Author: tedyu 
Authored: Mon Nov 21 13:00:27 2016 -0800
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:31 2017 -0400

--
 .../hbase/quotas/QuotaSettingsFactory.java  | 20 
 .../hadoop/hbase/quotas/SpaceLimitSettings.java |  8 
 .../hbase/shaded/protobuf/ProtobufUtil.java |  9 +
 3 files changed, 21 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/988a23ef/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 8512e39..7f1c180 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -128,7 +128,8 @@ public class QuotaSettingsFactory {
 
   static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota 
protoQuota) {
 if ((null == table && null == namespace) || (null != table && null != 
namespace)) {
-  throw new IllegalArgumentException("Can only construct 
SpaceLimitSettings for a table or namespace.");
+  throw new IllegalArgumentException(
+  "Can only construct SpaceLimitSettings for a table or namespace.");
 }
 if (null != table) {
   return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
@@ -300,29 +301,32 @@ public class QuotaSettingsFactory {
*/
 
   /**
-   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given table to the given size in bytes.
-   * When the space usage is exceeded by the table, the provided {@link 
SpaceViolationPolicy} is enacted on the table.
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given table
+   * to the given size in bytes. When the space usage is exceeded by the 
table, the provided
+   * {@link SpaceViolationPolicy} is enacted on the table.
*
* @param tableName The name of the table on which the quota should be 
applied.
* @param sizeLimit The limit of a table's size in bytes.
* @param violationPolicy The action to take when the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
-  public static QuotaSettings limitTableSpace(final TableName tableName, long 
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+  public static QuotaSettings limitTableSpace(
+  final TableName tableName, long sizeLimit, final SpaceViolationPolicy 
violationPolicy) {
 return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
   }
 
   /**
-   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given namespace to the given size in bytes.
-   * When the space usage is exceeded by all tables in the namespace, the 
provided {@link SpaceViolationPolicy} is enacted on
-   * all tables in the namespace.
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given
+   * namespace to the given size in bytes. When the space usage is exceeded by 
all tables in the
+   * namespace, the provided {@link SpaceViolationPolicy} is enacted on all 
tables in the namespace.
*
* @param namespace The namespace on which the quota should be applied.
* @param sizeLimit The limit of the namespace's size in bytes.
* @param violationPolicy The action to take when the the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
-  public static QuotaSettings limitNamespaceSpace(final String namespace, long 
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+  public static QuotaSettings limitNamespaceSpace(
+  final String namespace, long sizeLimit, final SpaceViolationPolicy 
violationPolicy) {
 return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/988a23ef/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git 

[50/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas

[elserj]

HBASE-17002 JMX metrics and some UI additions for space quotas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/96c6b8fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/96c6b8fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/96c6b8fa

Branch: refs/heads/HBASE-16961
Commit: 96c6b8fa8c7c038ee9b0134fb92061b0e61a5fd4
Parents: 71e14a3
Author: Josh Elser 
Authored: Wed Feb 15 14:24:57 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:47:49 2017 -0400

--
 .../hbase/client/ConnectionImplementation.java  |8 +
 .../hadoop/hbase/client/QuotaStatusCalls.java   |   39 +-
 .../client/ShortCircuitMasterConnection.java|8 +
 .../hadoop/hbase/quotas/QuotaTableUtil.java |   41 +
 .../hbase/shaded/protobuf/RequestConverter.java |   11 +
 .../hbase/master/MetricsMasterQuotaSource.java  |   75 +
 .../master/MetricsMasterQuotaSourceFactory.java |   26 +
 .../hbase/master/MetricsMasterWrapper.java  |   13 +
 .../MetricsRegionServerQuotaSource.java |   54 +
 .../MetricsMasterQuotaSourceFactoryImpl.java|   36 +
 .../master/MetricsMasterQuotaSourceImpl.java|  129 +
 ...hadoop.hbase.master.MetricsMasterQuotaSource |   18 +
 ...hbase.master.MetricsMasterQuotaSourceFactory |   18 +
 .../shaded/protobuf/generated/MasterProtos.java |   93 +-
 .../shaded/protobuf/generated/QuotaProtos.java  | 3099 +-
 .../src/main/protobuf/Master.proto  |6 +-
 .../src/main/protobuf/Quota.proto   |   17 +
 .../org/apache/hadoop/hbase/master/HMaster.java |2 +-
 .../hadoop/hbase/master/MasterRpcServices.java  |   38 +
 .../hadoop/hbase/master/MetricsMaster.java  |   42 +
 .../hbase/master/MetricsMasterWrapperImpl.java  |   42 +-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |   92 +-
 .../resources/hbase-webapps/master/table.jsp|   59 +
 .../hbase/master/TestMasterMetricsWrapper.java  |   17 +
 .../hbase/quotas/TestQuotaStatusRPCs.java   |   83 +
 25 files changed, 4032 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 3f27e1c..d9219f3 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -94,6 +94,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
@@ -1740,6 +1742,12 @@ class ConnectionImplementation implements 
ClusterConnection, Closeable {
   throws ServiceException {
 return stub.getSpaceQuotaRegionSizes(controller, request);
   }
+
+  @Override
+  public GetQuotaStatesResponse getQuotaStates(
+  RpcController controller, GetQuotaStatesRequest request) throws 
ServiceException {
+return stub.getQuotaStates(controller, request);
+  }
 };
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
index f0f385d..af36d1e 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 

[34/50] [abbrv] hbase git commit: HBASE-17478 Avoid reporting FS use when quotas are disabled

[elserj]

HBASE-17478 Avoid reporting FS use when quotas are disabled

Also, gracefully produce responses when quotas are disabled.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/efd6edc4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/efd6edc4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/efd6edc4

Branch: refs/heads/HBASE-16961
Commit: efd6edc44193155fabbf8dfe8d7af360087023c1
Parents: 7a1f9d3
Author: Josh Elser 
Authored: Tue Jan 17 14:41:45 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:32 2017 -0400

--
 .../hadoop/hbase/master/MasterRpcServices.java  |  4 +++
 .../hadoop/hbase/quotas/MasterQuotaManager.java | 13 +--
 .../hbase/regionserver/HRegionServer.java   |  5 ++-
 .../hbase/quotas/TestMasterQuotaManager.java| 37 
 4 files changed, 56 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/efd6edc4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index f454248..de437fd 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -59,6 +59,7 @@ import 
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
 import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaUtil;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -2016,6 +2017,9 @@ public class MasterRpcServices extends RSRpcServices
   RegionSpaceUseReportRequest request) throws ServiceException {
 try {
   master.checkInitialized();
+  if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) {
+return RegionSpaceUseReportResponse.newBuilder().build();
+  }
   MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
   for (RegionSpaceUse report : request.getSpaceUseList()) {
 quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize());

http://git-wip-us.apache.org/repos/asf/hbase/blob/efd6edc4/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index a5832f9..cb614ea 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.quotas;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -58,6 +59,8 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
 @InterfaceStability.Evolving
 public class MasterQuotaManager implements RegionStateListener {
   private static final Log LOG = LogFactory.getLog(MasterQuotaManager.class);
+  private static final Map EMPTY_MAP = 
Collections.unmodifiableMap(
+  new HashMap<>());
 
   private final MasterServices masterServices;
   private NamedLock namespaceLocks;
@@ -529,13 +532,19 @@ public class MasterQuotaManager implements 
RegionStateListener {
   }
 
   public void addRegionSize(HRegionInfo hri, long size) {
-// TODO Make proper API
+if (null == regionSizes) {
+  return;
+}
+// TODO Make proper API?
 // TODO Prevent from growing indefinitely
 regionSizes.put(hri, size);
   }
 
   public Map snapshotRegionSizes() {
-// TODO Make proper API
+if (null == regionSizes) {
+  return EMPTY_MAP;
+}
+// TODO Make proper API?
 return new HashMap<>(regionSizes);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/efd6edc4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 

[03/50] [abbrv] hbase git commit: HBASE-17090 Redundant exclusion of jruby-complete in pom of hbase-spark

[elserj]

HBASE-17090 Redundant exclusion of jruby-complete in pom of hbase-spark

Signed-off-by: Michael Stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e2a74615
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e2a74615
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e2a74615

Branch: refs/heads/HBASE-16961
Commit: e2a746152ca8c02c18214f0b5180ed8dcc84e947
Parents: 9dd5cda
Author: Xiang Li 
Authored: Fri Apr 14 16:15:42 2017 +0800
Committer: Michael Stack 
Committed: Fri Apr 14 08:08:42 2017 -0700

--
 hbase-spark/pom.xml | 24 
 1 file changed, 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e2a74615/hbase-spark/pom.xml
--
diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml
index a7997f1..1afae85 100644
--- a/hbase-spark/pom.xml
+++ b/hbase-spark/pom.xml
@@ -290,10 +290,6 @@
 thrift
 
 
-org.jruby
-jruby-complete
-
-
 org.slf4j
 slf4j-log4j12
 
@@ -338,10 +334,6 @@
 jasper-compiler
 
 
-org.jruby
-jruby-complete
-
-
 org.jboss.netty
 netty
 
@@ -382,10 +374,6 @@
 thrift
 
 
-org.jruby
-jruby-complete
-
-
 org.slf4j
 slf4j-log4j12
 
@@ -430,10 +418,6 @@
 jasper-compiler
 
 
-org.jruby
-jruby-complete
-
-
 org.jboss.netty
 netty
 
@@ -460,10 +444,6 @@
 thrift
 
 
-org.jruby
-jruby-complete
-
-
 org.slf4j
 slf4j-log4j12
 
@@ -508,10 +488,6 @@
 jasper-compiler
 
 
-org.jruby
-jruby-complete
-
-
 org.jboss.netty
 netty
 

[36/50] [abbrv] hbase git commit: HBASE-16999 Implement master and regionserver synchronization of quota state

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/dccfc846/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
new file mode 100644
index 000..4a7000c
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Test case for {@link TableSpaceQuotaViolationNotifier}.
+ */
+@Category(SmallTests.class)
+public class TestTableSpaceQuotaViolationNotifier {
+
+  private TableSpaceQuotaViolationNotifier notifier;
+  private Connection conn;
+
+  @Before
+  public void setup() throws Exception {
+notifier = new TableSpaceQuotaViolationNotifier();
+conn = mock(Connection.class);
+notifier.initialize(conn);
+  }
+
+  @Test
+  public void testToViolation() throws Exception {
+final TableName tn = TableName.valueOf("inviolation");
+final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Put expectedPut = new Put(Bytes.toBytes("t." + 
tn.getNameAsString()));
+final SpaceQuota protoQuota = SpaceQuota.newBuilder()
+.setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(policy))
+.build();
+expectedPut.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"), 
protoQuota.toByteArray());
+
+notifier.transitionTableToViolation(tn, policy);
+
+verify(quotaTable).put(argThat(new SingleCellPutMatcher(expectedPut)));
+  }
+
+  @Test
+  public void testToObservance() throws Exception {
+final TableName tn = TableName.valueOf("notinviolation");
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Delete expectedDelete = new Delete(Bytes.toBytes("t." + 
tn.getNameAsString()));
+expectedDelete.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"));
+
+notifier.transitionTableToObservance(tn);
+
+verify(quotaTable).delete(argThat(new 
SingleCellDeleteMatcher(expectedDelete)));
+  }
+
+  /**
+   * Parameterized for Puts.
+   */
+  private static class SingleCellPutMatcher extends 
SingleCellMutationMatcher {
+private SingleCellPutMatcher(Put expected) {
+  super(expected);
+}
+  }
+
+  /**
+   * Parameterized for Deletes.
+   */
+  private static class SingleCellDeleteMatcher extends 
SingleCellMutationMatcher {
+private SingleCellDeleteMatcher(Delete expected) {
+  super(expected);
+}
+  }
+
+  /**
+   * Quick hack to verify a Mutation with one column.
+   */
+  private static class SingleCellMutationMatcher extends ArgumentMatcher 
{
+private final Mutation expected;
+
+private SingleCellMutationMatcher(Mutation expected) {
+  

[16/50] [abbrv] hbase git commit: HBASE-17557 HRegionServer#reportRegionSizesForQuotas() should respond to UnsupportedOperationException

[elserj]

HBASE-17557 HRegionServer#reportRegionSizesForQuotas() should respond to 
UnsupportedOperationException


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d77ec498
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d77ec498
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d77ec498

Branch: refs/heads/HBASE-16961
Commit: d77ec498c23de9c75390237ce028fe6e32f3c8b3
Parents: 2dea676
Author: tedyu 
Authored: Mon Jan 30 07:47:40 2017 -0800
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:31 2017 -0400

--
 .../quotas/FileSystemUtilizationChore.java  | 20 +---
 .../hbase/regionserver/HRegionServer.java   | 24 
 2 files changed, 36 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d77ec498/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
index 01540eb..efc17ff 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -53,6 +53,9 @@ public class FileSystemUtilizationChore extends 
ScheduledChore {
   static final String FS_UTILIZATION_MAX_ITERATION_DURATION_KEY = 
"hbase.regionserver.quotas.fs.utilization.chore.max.iteration.millis";
   static final long FS_UTILIZATION_MAX_ITERATION_DURATION_DEFAULT = 5000L;
 
+  private int numberOfCyclesToSkip = 0, prevNumberOfCyclesToSkip = 0;
+  private static final int CYCLE_UPPER_BOUND = 32;
+
   private final HRegionServer rs;
   private final long maxIterationMillis;
   private Iterator leftoverRegions;
@@ -67,6 +70,10 @@ public class FileSystemUtilizationChore extends 
ScheduledChore {
 
   @Override
   protected void chore() {
+if (numberOfCyclesToSkip > 0) {
+  numberOfCyclesToSkip--;
+  return;
+}
 final Map onlineRegionSizes = new HashMap<>();
 final Set onlineRegions = new HashSet<>(rs.getOnlineRegions());
 // Process the regions from the last run if we have any. If we are somehow 
having difficulty
@@ -126,7 +133,14 @@ public class FileSystemUtilizationChore extends 
ScheduledChore {
   + skippedSplitParents + " regions due to being the parent of a 
split, and"
   + skippedRegionReplicas + " regions due to being region replicas.");
 }
-reportRegionSizesToMaster(onlineRegionSizes);
+if (!reportRegionSizesToMaster(onlineRegionSizes)) {
+  // backoff reporting
+  numberOfCyclesToSkip = prevNumberOfCyclesToSkip > 0 ? 2 * 
prevNumberOfCyclesToSkip : 1;
+  if (numberOfCyclesToSkip > CYCLE_UPPER_BOUND) {
+numberOfCyclesToSkip = CYCLE_UPPER_BOUND;
+  }
+  prevNumberOfCyclesToSkip = numberOfCyclesToSkip;
+}
   }
 
   /**
@@ -166,8 +180,8 @@ public class FileSystemUtilizationChore extends 
ScheduledChore {
*
* @param onlineRegionSizes The computed region sizes to report.
*/
-  void reportRegionSizesToMaster(Map onlineRegionSizes) {
-this.rs.reportRegionSizesForQuotas(onlineRegionSizes);
+  boolean reportRegionSizesToMaster(Map onlineRegionSizes) {
+return this.rs.reportRegionSizesForQuotas(onlineRegionSizes);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/d77ec498/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9be4131..053e4ac 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.ChoreService;
 import org.apache.hadoop.hbase.ClockOutOfSyncException;
 import org.apache.hadoop.hbase.CoordinatedStateManager;
 import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HConstants;
@@ -1248,13 +1249,14 @@ public class HRegionServer extends HasThread implements
* Reports the given map of Regions and their size on the filesystem to the 
active Master.
*
* 

[44/50] [abbrv] hbase git commit: HBASE-17516 Correctly handle case where table and NS quotas both apply

[elserj]

HBASE-17516 Correctly handle case where table and NS quotas both apply

The logic surrounding when a table and namespace quota both apply
to a table was incorrect, leading to a case where a table quota
violation which should have fired did not because of the less-strict
namespace quota.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/48332eeb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/48332eeb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/48332eeb

Branch: refs/heads/HBASE-16961
Commit: 48332eebfd7fdd0a2065db448e82e82e1548cfc4
Parents: 095fabf
Author: Josh Elser 
Authored: Wed Feb 22 18:32:55 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:44:00 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaObserverChore.java | 10 ++-
 .../TestQuotaObserverChoreWithMiniCluster.java  | 66 
 .../hbase/quotas/TestQuotaStatusRPCs.java   | 21 ++-
 .../hadoop/hbase/quotas/TestSpaceQuotas.java| 32 +-
 4 files changed, 97 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/48332eeb/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 973ac8c..b9f4592 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -287,7 +287,8 @@ public class QuotaObserverChore extends ScheduledChore {
   // We want to have a policy of "NONE", moving out of violation
   if (!targetStatus.isInViolation()) {
 for (TableName tableInNS : tablesByNamespace.get(namespace)) {
-  if 
(!tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
 {
+  // If there is a quota on this table in violation
+  if 
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
 {
 // Table-level quota violation policy is being applied here.
 if (LOG.isTraceEnabled()) {
   LOG.trace("Not activating Namespace violation policy because a 
Table violation"
@@ -298,16 +299,21 @@ public class QuotaObserverChore extends ScheduledChore {
 this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
   }
 }
+  // We want to move into violation at the NS level
   } else {
 // Moving tables in the namespace into violation or to a different 
violation policy
 for (TableName tableInNS : tablesByNamespace.get(namespace)) {
-  if 
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
 {
+  final SpaceQuotaSnapshot tableQuotaSnapshot =
+tableSnapshotStore.getCurrentState(tableInNS);
+  final boolean hasTableQuota = QuotaSnapshotStore.NO_QUOTA != 
tableQuotaSnapshot;
+  if (hasTableQuota && 
tableQuotaSnapshot.getQuotaStatus().isInViolation()) {
 // Table-level quota violation policy is being applied here.
 if (LOG.isTraceEnabled()) {
   LOG.trace("Not activating Namespace violation policy because a 
Table violation"
   + " policy is already in effect for " + tableInNS);
 }
   } else {
+// No table quota present or a table quota present that is not in 
violation
 LOG.info(tableInNS + " moving into violation of namespace space 
quota with policy " + targetStatus.getPolicy());
 this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/48332eeb/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index 943c898..63198a8 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -193,40 +193,42 @@ public class TestQuotaObserverChoreWithMiniCluster {
 
 helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
 admin.flush(tn1);
-Map violatedQuotas = 

[15/50] [abbrv] hbase git commit: Revert "HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)"

[elserj]

Revert "HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)"

This reverts commit c2c2178b2eebe4439eadec6b37fae2566944c16b.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ecdfb823
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ecdfb823
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ecdfb823

Branch: refs/heads/HBASE-16961
Commit: ecdfb82326035ad8221940919bbeb3fe16ec2658
Parents: c2c2178
Author: Ramkrishna 
Authored: Tue Apr 18 00:00:12 2017 +0530
Committer: Ramkrishna 
Committed: Tue Apr 18 00:00:12 2017 +0530

--
 .../java/org/apache/hadoop/hbase/CellUtil.java  |  24 ++
 .../org/apache/hadoop/hbase/ExtendedCell.java   |  10 -
 .../org/apache/hadoop/hbase/master/HMaster.java |   2 -
 .../hbase/regionserver/ByteBufferChunkCell.java |  48 ---
 .../apache/hadoop/hbase/regionserver/Chunk.java |  60 +--
 .../hadoop/hbase/regionserver/ChunkCreator.java | 404 ---
 .../hbase/regionserver/HRegionServer.java   |  14 +-
 .../hbase/regionserver/MemStoreChunkPool.java   | 265 
 .../hadoop/hbase/regionserver/MemStoreLAB.java  |   4 +-
 .../hbase/regionserver/MemStoreLABImpl.java | 171 
 .../regionserver/NoTagByteBufferChunkCell.java  |  48 ---
 .../hadoop/hbase/regionserver/OffheapChunk.java |  31 +-
 .../hadoop/hbase/regionserver/OnheapChunk.java  |  32 +-
 .../hadoop/hbase/HBaseTestingUtility.java   |   3 -
 .../coprocessor/TestCoprocessorInterface.java   |   4 -
 .../TestRegionObserverScannerOpenHook.java  |   3 -
 .../coprocessor/TestRegionObserverStacking.java |   3 -
 .../io/hfile/TestScannerFromBucketCache.java|   3 -
 .../hadoop/hbase/master/TestCatalogJanitor.java |   7 -
 .../hadoop/hbase/regionserver/TestBulkLoad.java |   2 +-
 .../hbase/regionserver/TestCellFlatSet.java |   2 +-
 .../regionserver/TestCompactingMemStore.java|  37 +-
 .../TestCompactingToCellArrayMapMemStore.java   |  16 +-
 .../TestCompactionArchiveConcurrentClose.java   |   1 -
 .../TestCompactionArchiveIOException.java   |   1 -
 .../regionserver/TestCompactionPolicy.java  |   1 -
 .../hbase/regionserver/TestDefaultMemStore.java |  14 +-
 .../regionserver/TestFailedAppendAndSync.java   |   1 -
 .../hbase/regionserver/TestHMobStore.java   |   2 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |   2 -
 .../regionserver/TestHRegionReplayEvents.java   |   2 +-
 .../regionserver/TestMemStoreChunkPool.java |  48 +--
 .../hbase/regionserver/TestMemStoreLAB.java |  27 +-
 .../TestMemstoreLABWithoutPool.java | 168 
 .../hbase/regionserver/TestRecoveredEdits.java  |   1 -
 .../hbase/regionserver/TestRegionIncrement.java |   1 -
 .../hadoop/hbase/regionserver/TestStore.java|   1 -
 .../TestStoreFileRefresherChore.java|   1 -
 .../hbase/regionserver/TestWALLockup.java   |   1 -
 .../TestWALMonotonicallyIncreasingSeqId.java|   1 -
 .../hbase/regionserver/wal/TestDurability.java  |   3 -
 41 files changed, 479 insertions(+), 990 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index 56de21b..e1bc969 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -3135,4 +3135,28 @@ public final class CellUtil {
   return Type.DeleteFamily.getCode();
 }
   }
+
+  /**
+   * Clone the passed cell by copying its data into the passed buf.
+   */
+  public static Cell copyCellTo(Cell cell, ByteBuffer buf, int offset, int 
len) {
+int tagsLen = cell.getTagsLength();
+if (cell instanceof ExtendedCell) {
+  ((ExtendedCell) cell).write(buf, offset);
+} else {
+  // Normally all Cell impls within Server will be of type ExtendedCell. 
Just considering the
+  // other case also. The data fragments within Cell is copied into buf as 
in KeyValue
+  // serialization format only.
+  KeyValueUtil.appendTo(cell, buf, offset, true);
+}
+if (tagsLen == 0) {
+  // When tagsLen is 0, make a NoTagsByteBufferKeyValue version. This is 
an optimized class
+  // which directly return tagsLen as 0. So we avoid parsing many length 
components in
+  // reading the tagLength stored in the backing buffer. The Memstore 
addition of every Cell
+  // call getTagsLength().
+  return new NoTagsByteBufferKeyValue(buf, offset, len, 
cell.getSequenceId());
+} else {
+  return new 

[11/50] [abbrv] hbase git commit: HBASE-16875 Changed try-with-resources in the docs to recommended way

[elserj]

HBASE-16875 Changed try-with-resources in the docs to recommended way

Signed-off-by: Chia-Ping Tsai 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c8cd921b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c8cd921b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c8cd921b

Branch: refs/heads/HBASE-16961
Commit: c8cd921bededa67b2b0de823005830d750534d93
Parents: c1ac3f7
Author: Jan Hentschel 
Authored: Sat Mar 4 10:04:02 2017 +0100
Committer: Chia-Ping Tsai 
Committed: Mon Apr 17 10:59:46 2017 +0800

--
 src/main/asciidoc/_chapters/architecture.adoc |  7 +++---
 src/main/asciidoc/_chapters/security.adoc | 29 --
 2 files changed, 13 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c8cd921b/src/main/asciidoc/_chapters/architecture.adoc
--
diff --git a/src/main/asciidoc/_chapters/architecture.adoc 
b/src/main/asciidoc/_chapters/architecture.adoc
index 27aebd9..7f9ba07 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -219,10 +219,9 @@ For applications which require high-end multithreaded 
access (e.g., web-servers
 
 // Create a connection to the cluster.
 Configuration conf = HBaseConfiguration.create();
-try (Connection connection = ConnectionFactory.createConnection(conf)) {
-  try (Table table = connection.getTable(TableName.valueOf(tablename)) {
-// use table as needed, the table returned is lightweight
-  }
+try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tablename))) {
+  // use table as needed, the table returned is lightweight
 }
 
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c8cd921b/src/main/asciidoc/_chapters/security.adoc
--
diff --git a/src/main/asciidoc/_chapters/security.adoc 
b/src/main/asciidoc/_chapters/security.adoc
index 0ed9ba2..ccb5adb 100644
--- a/src/main/asciidoc/_chapters/security.adoc
+++ b/src/main/asciidoc/_chapters/security.adoc
@@ -202,10 +202,9 @@ Set it in the `Configuration` supplied to `Table`:
 Configuration conf = HBaseConfiguration.create();
 Connection connection = ConnectionFactory.createConnection(conf);
 conf.set("hbase.rpc.protection", "privacy");
-try (Connection connection = ConnectionFactory.createConnection(conf)) {
-  try (Table table = connection.getTable(TableName.valueOf(tablename)) {
+try (Connection connection = ConnectionFactory.createConnection(conf);
+ Table table = connection.getTable(TableName.valueOf(tablename))) {
    do your stuff
-  }
 }
 
 
@@ -1014,24 +1013,16 @@ public static void grantOnTable(final 
HBaseTestingUtility util, final String use
   SecureTestUtil.updateACLs(util, new Callable() {
 @Override
 public Void call() throws Exception {
-  Configuration conf = HBaseConfiguration.create();
-  Connection connection = ConnectionFactory.createConnection(conf);
-  try (Connection connection = ConnectionFactory.createConnection(conf)) {
-try (Table table = connection.getTable(TableName.valueOf(tablename)) {
-  AccessControlLists.ACL_TABLE_NAME);
-  try {
-BlockingRpcChannel service = 
acl.coprocessorService(HConstants.EMPTY_START_ROW);
-AccessControlService.BlockingInterface protocol =
-AccessControlService.newBlockingStub(service);
-ProtobufUtil.grant(protocol, user, table, family, qualifier, 
actions);
-  } finally {
-acl.close();
-  }
-  return null;
-}
+  try (Connection connection = 
ConnectionFactory.createConnection(util.getConfiguration());
+   Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) 
{
+BlockingRpcChannel service = 
acl.coprocessorService(HConstants.EMPTY_START_ROW);
+AccessControlService.BlockingInterface protocol =
+  AccessControlService.newBlockingStub(service);
+AccessControlUtil.grant(null, protocol, user, table, family, 
qualifier, false, actions);
   }
+  return null;
 }
-  }
+  });
 }
 
 

[10/50] [abbrv] hbase git commit: HBASE-17366 Run TestHFile#testReaderWithoutBlockCache failes

[elserj]

HBASE-17366 Run TestHFile#testReaderWithoutBlockCache failes

Signed-off-by: CHIA-PING TSAI 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1ac3f77
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1ac3f77
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1ac3f77

Branch: refs/heads/HBASE-16961
Commit: c1ac3f7739f8c9e20f6aed428558128339467d04
Parents: 363f627
Author: huaxiang sun 
Authored: Mon Apr 17 10:32:17 2017 +0800
Committer: CHIA-PING TSAI 
Committed: Mon Apr 17 10:34:17 2017 +0800

--
 .../apache/hadoop/hbase/regionserver/StoreFileWriter.java   | 9 +
 .../java/org/apache/hadoop/hbase/io/hfile/TestHFile.java| 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c1ac3f77/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
index ccfd735..88cba75 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
@@ -384,6 +384,15 @@ public class StoreFileWriter implements CellSink, 
ShipperListener {
 }
 
 /**
+ * Creates Builder with cache configuration disabled
+ */
+public Builder(Configuration conf, FileSystem fs) {
+  this.conf = conf;
+  this.cacheConf = CacheConfig.DISABLED;
+  this.fs = fs;
+}
+
+/**
  * @param trt A premade TimeRangeTracker to use rather than build one per 
append (building one
  * of these is expensive so good to pass one in if you have one).
  * @return this (for chained invocation)

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1ac3f77/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
index 7074c9d..4db459a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
@@ -115,7 +115,7 @@ public class TestHFile  {
 Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), 
"TestHFile");
 HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 
1024).build();
 StoreFileWriter sfw =
-new StoreFileWriter.Builder(conf, cacheConf, 
fs).withOutputDir(storeFileParentDir)
+new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
 
.withComparator(CellComparator.COMPARATOR).withFileContext(meta).build();
 
 final int rowLen = 32;

[38/50] [abbrv] hbase git commit: HBASE-17025 Add shell commands for space quotas

[elserj]

HBASE-17025 Add shell commands for space quotas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7a1f9d34
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7a1f9d34
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7a1f9d34

Branch: refs/heads/HBASE-16961
Commit: 7a1f9d34b89131f248f76d1fafde6b060bec2827
Parents: 0effca4
Author: Josh Elser 
Authored: Wed Jan 11 11:55:29 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:32 2017 -0400

--
 hbase-shell/src/main/ruby/hbase/quotas.rb   |  62 -
 hbase-shell/src/main/ruby/hbase_constants.rb|   1 +
 .../src/main/ruby/shell/commands/set_quota.rb   |  45 +-
 .../hadoop/hbase/client/AbstractTestShell.java  |   1 +
 hbase-shell/src/test/ruby/hbase/quotas_test.rb  | 137 +++
 hbase-shell/src/test/ruby/tests_runner.rb   |   1 +
 6 files changed, 242 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7a1f9d34/hbase-shell/src/main/ruby/hbase/quotas.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb 
b/hbase-shell/src/main/ruby/hbase/quotas.rb
index bf2dc63..d99fe72 100644
--- a/hbase-shell/src/main/ruby/hbase/quotas.rb
+++ b/hbase-shell/src/main/ruby/hbase/quotas.rb
@@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
 java_import org.apache.hadoop.hbase.quotas.QuotaFilter
 java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
 java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
+java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
 
 module HBaseQuotasConstants
+  # RPC Quota constants
   GLOBAL_BYPASS = 'GLOBAL_BYPASS'
   THROTTLE_TYPE = 'THROTTLE_TYPE'
   THROTTLE = 'THROTTLE'
   REQUEST = 'REQUEST'
   WRITE = 'WRITE'
   READ = 'READ'
+  # Space quota constants
+  SPACE = 'SPACE'
+  NO_INSERTS = 'NO_INSERTS'
+  NO_WRITES = 'NO_WRITES'
+  NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
+  DISABLE = 'DISABLE'
 end
 
 module Hbase
@@ -107,6 +115,54 @@ module Hbase
   @admin.setQuota(settings)
 end
 
+def limit_space(args)
+  raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? 
and args.kind_of?(Hash))
+  # Let the user provide a raw number
+  if args[LIMIT].is_a?(Numeric)
+limit = args[LIMIT]
+  else
+# Parse a string a 1K, 2G, etc.
+limit = _parse_size(args[LIMIT])
+  end
+  # Extract the policy, failing if something bogus was provided
+  policy = SpaceViolationPolicy.valueOf(args[POLICY])
+  # Create a table or namespace quota
+  if args.key?(TABLE)
+if args.key?(NAMESPACE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+settings = 
QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)), 
limit, policy)
+  elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+settings = 
QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy)
+  else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+  end
+  # Apply the quota
+  @admin.setQuota(settings)
+end
+
+def remove_space_limit(args)
+  raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? 
and args.kind_of?(Hash))
+  if args.key?(TABLE)
+if args.key?(NAMESPACE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+table = TableName.valueOf(args.delete(TABLE))
+settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
+  elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+  raise(ArgumentError, "Only one of TABLE or NAMESPACE can be 
specified.")
+end
+settings = 
QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
+  else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+  end
+  @admin.setQuota(settings)
+end
+
 def set_global_bypass(bypass, args)
   raise(ArgumentError, "Arguments should be a Hash") unless 
args.kind_of?(Hash)
 
@@ -171,7 +227,7 @@ module Hbase
   return _size_from_str(match[1].to_i, match[2])
 end
   else
-raise "Invalid size limit syntax"
+raise(ArgumentError, "Invalid size limit syntax")
   end
 end
 
@@ -188,7 +244,7 @@ module Hbase
 end
 
 if limit <= 0
-  raise "Invalid throttle limit, must be greater then 0"
+  raise(ArgumentError, "Invalid throttle limit, must be greater then 
0")
 end
 
 

[24/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)

[elserj]

HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/990062a9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/990062a9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/990062a9

Branch: refs/heads/HBASE-16961
Commit: 990062a93a257a42adc84b7b8448d788517c9baa
Parents: ecdfb82
Author: tedyu 
Authored: Thu Nov 17 10:19:52 2016 -0800
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:31 2017 -0400

--
 .../hbase/quotas/QuotaSettingsFactory.java  |   47 +
 .../apache/hadoop/hbase/quotas/QuotaType.java   |1 +
 .../hadoop/hbase/quotas/SpaceLimitSettings.java |  166 ++
 .../hbase/quotas/SpaceViolationPolicy.java  |   44 +
 .../hbase/shaded/protobuf/ProtobufUtil.java |   51 +
 .../hbase/quotas/TestQuotaSettingsFactory.java  |  148 ++
 .../hbase/quotas/TestSpaceLimitSettings.java|  119 ++
 .../shaded/protobuf/generated/MasterProtos.java |  584 --
 .../shaded/protobuf/generated/QuotaProtos.java  | 1739 +-
 .../src/main/protobuf/Master.proto  |2 +
 .../src/main/protobuf/Quota.proto   |   21 +
 .../hbase/protobuf/generated/QuotaProtos.java   | 1682 -
 hbase-protocol/src/main/protobuf/Quota.proto|   21 +
 13 files changed, 4291 insertions(+), 334 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/990062a9/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 3622a32..8512e39 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -27,6 +27,7 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRe
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 
 @InterfaceAudience.Public
 public class QuotaSettingsFactory {
@@ -89,6 +90,9 @@ public class QuotaSettingsFactory {
 if (quotas.getBypassGlobals() == true) {
   settings.add(new QuotaGlobalsSettingsBypass(userName, tableName, 
namespace, true));
 }
+if (quotas.hasSpace()) {
+  settings.add(fromSpace(tableName, namespace, quotas.getSpace()));
+}
 return settings;
   }
 
@@ -122,6 +126,18 @@ public class QuotaSettingsFactory {
 return settings;
   }
 
+  static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota 
protoQuota) {
+if ((null == table && null == namespace) || (null != table && null != 
namespace)) {
+  throw new IllegalArgumentException("Can only construct 
SpaceLimitSettings for a table or namespace.");
+}
+if (null != table) {
+  return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
+} else {
+  // namespace must be non-null
+  return SpaceLimitSettings.fromSpaceQuota(namespace, protoQuota);
+}
+  }
+
   /* ==
*  RPC Throttle
*/
@@ -278,4 +294,35 @@ public class QuotaSettingsFactory {
   public static QuotaSettings bypassGlobals(final String userName, final 
boolean bypassGlobals) {
 return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals);
   }
+
+  /* ==
+   *  FileSystem Space Settings
+   */
+
+  /**
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given table to the given size in bytes.
+   * When the space usage is exceeded by the table, the provided {@link 
SpaceViolationPolicy} is enacted on the table.
+   *
+   * @param tableName The name of the table on which the quota should be 
applied.
+   * @param sizeLimit The limit of a table's size in bytes.
+   * @param violationPolicy The action to take when the quota is exceeded.
+   * @return An {@link QuotaSettings} object.
+   */
+  public static QuotaSettings limitTableSpace(final TableName tableName, long 
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
+  }
+
+  /**
+   * Creates a {@link QuotaSettings} object to limit the FileSystem space 
usage for the given namespace to the given size in bytes.
+  

[29/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index c493b25..943c898 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -22,16 +22,12 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
@@ -40,20 +36,15 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -62,7 +53,6 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
 
-import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Multimap;
 
@@ -72,11 +62,8 @@ import com.google.common.collect.Multimap;
 @Category(LargeTests.class)
 public class TestQuotaObserverChoreWithMiniCluster {
   private static final Log LOG = 
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
-  private static final int SIZE_PER_VALUE = 256;
-  private static final String F1 = "f1";
   private static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
   private static final AtomicLong COUNTER = new AtomicLong(0);
-  private static final long ONE_MEGABYTE = 1024L * 1024L;
   private static final long DEFAULT_WAIT_MILLIS = 500;
 
   @Rule
@@ -84,18 +71,19 @@ public class TestQuotaObserverChoreWithMiniCluster {
 
   private HMaster master;
   private QuotaObserverChore chore;
-  private SpaceQuotaViolationNotifierForTest violationNotifier;
+  private SpaceQuotaSnapshotNotifierForTest snapshotNotifier;
+  private SpaceQuotaHelperForTests helper;
 
   @BeforeClass
   public static void setUp() throws Exception {
 Configuration conf = TEST_UTIL.getConfiguration();
 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 
1000);
 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 
1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
-conf.setClass(SpaceQuotaViolationNotifierFactory.VIOLATION_NOTIFIER_KEY,
-SpaceQuotaViolationNotifierForTest.class, 
SpaceQuotaViolationNotifier.class);
+conf.setClass(SpaceQuotaSnapshotNotifierFactory.SNAPSHOT_NOTIFIER_KEY,
+SpaceQuotaSnapshotNotifierForTest.class, 
SpaceQuotaSnapshotNotifier.class);
 TEST_UTIL.startMiniCluster(1);
   }
 
@@ -131,40 +119,55 @@ public class TestQuotaObserverChoreWithMiniCluster {
 }
 
 master = TEST_UTIL.getMiniHBaseCluster().getMaster();
-violationNotifier =
-(SpaceQuotaViolationNotifierForTest) 
master.getSpaceQuotaViolationNotifier();
-violationNotifier.clearTableViolations();
+snapshotNotifier =
+(SpaceQuotaSnapshotNotifierForTest) 
master.getSpaceQuotaSnapshotNotifier();
+snapshotNotifier.clearSnapshots();
 chore = master.getQuotaObserverChore();
+

[01/50] [abbrv] hbase git commit: HBASE-16775 Fix flaky TestExportSnapshot#testExportRetry. [Forced Update!]

[elserj]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-16961 f46c16aa2 -> 2524f8cd2 (forced update)


HBASE-16775 Fix flaky TestExportSnapshot#testExportRetry.

Reason for flakyness: Current test is probability based fault injection and 
triggers failure 3% of the time. Earlier when test used LocalJobRunner which 
didn't honor "mapreduce.map.maxattempts", it'd pass 97% time (when no fault is 
injected) and fail 3% time (when fault was injected). Point being, even when 
the test was complete wrong, we couldn't catch it because it was probability 
based.

This change will inject fault in a deterministic manner.
On design side, it encapsulates all testing hooks in ExportSnapshot.java into 
single inner class.

Change-Id: Icba866e1d56a5281748df89f4dd374bc45bad249


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da5fb27e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da5fb27e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da5fb27e

Branch: refs/heads/HBASE-16961
Commit: da5fb27eabed4a4b4d251be973ee945fb52895bf
Parents: cf3215d
Author: Apekshit Sharma 
Authored: Thu Oct 6 14:20:58 2016 -0700
Committer: Apekshit Sharma 
Committed: Wed Apr 12 11:11:31 2017 -0700

--
 .../hadoop/hbase/snapshot/ExportSnapshot.java   | 58 +++---
 .../hbase/snapshot/TestExportSnapshot.java  | 84 +++-
 .../snapshot/TestExportSnapshotNoCluster.java   |  2 +-
 .../hbase/snapshot/TestMobExportSnapshot.java   |  7 +-
 .../snapshot/TestMobSecureExportSnapshot.java   |  7 +-
 .../snapshot/TestSecureExportSnapshot.java  |  7 +-
 6 files changed, 93 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/da5fb27e/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
index e2086e9..e3ad951 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java
@@ -29,7 +29,6 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Random;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
@@ -110,9 +109,12 @@ public class ExportSnapshot extends AbstractHBaseTool 
implements Tool {
   private static final String CONF_BANDWIDTH_MB = 
"snapshot.export.map.bandwidth.mb";
   protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
 
-  static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
-  static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
-
+  static class Testing {
+static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
+static final String CONF_TEST_FAILURE_COUNT = 
"test.snapshot.export.failure.count";
+int failuresCountToInject = 0;
+int injectedFailureCount = 0;
+  }
 
   // Command line options and defaults.
   static final class Options {
@@ -149,12 +151,10 @@ public class ExportSnapshot extends AbstractHBaseTool 
implements Tool {
 
   private static class ExportMapper extends Mapper 
{
+private static final Log LOG = LogFactory.getLog(ExportMapper.class);
 final static int REPORT_SIZE = 1 * 1024 * 1024;
 final static int BUFFER_SIZE = 64 * 1024;
 
-private boolean testFailures;
-private Random random;
-
 private boolean verifyChecksum;
 private String filesGroup;
 private String filesUser;
@@ -169,9 +169,12 @@ public class ExportSnapshot extends AbstractHBaseTool 
implements Tool {
 private Path inputArchive;
 private Path inputRoot;
 
+private static Testing testing = new Testing();
+
 @Override
 public void setup(Context context) throws IOException {
   Configuration conf = context.getConfiguration();
+
   Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, 
CONF_SOURCE_PREFIX);
   Configuration destConf = HBaseConfiguration.createClusterConf(conf, 
null, CONF_DEST_PREFIX);
 
@@ -186,8 +189,6 @@ public class ExportSnapshot extends AbstractHBaseTool 
implements Tool {
   inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
   outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
 
-  testFailures = conf.getBoolean(CONF_TEST_FAILURE, false);
-
   try {
 srcConf.setBoolean("fs." + 

[37/50] [abbrv] hbase git commit: HBASE-16999 Implement master and regionserver synchronization of quota state

[elserj]

HBASE-16999 Implement master and regionserver synchronization of quota state

* Implement the RegionServer reading violation from the quota table
* Implement the Master reporting violations to the quota table
* RegionServers need to track its enforced policies


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dccfc846
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dccfc846
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dccfc846

Branch: refs/heads/HBASE-16961
Commit: dccfc8464e22c0a4c0efde50c7087db49a2ec1b0
Parents: bcf6da4
Author: Josh Elser 
Authored: Fri Nov 18 15:38:19 2016 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:32 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  92 -
 .../org/apache/hadoop/hbase/master/HMaster.java |  35 +++-
 .../hadoop/hbase/quotas/QuotaObserverChore.java |   5 +-
 .../hbase/quotas/RegionServerQuotaManager.java  | 200 ---
 .../quotas/RegionServerRpcQuotaManager.java | 200 +++
 .../quotas/RegionServerSpaceQuotaManager.java   | 169 
 .../quotas/SpaceQuotaViolationNotifier.java |  16 +-
 .../SpaceQuotaViolationNotifierFactory.java |  62 ++
 .../SpaceQuotaViolationNotifierForTest.java |   4 +
 ...SpaceQuotaViolationPolicyRefresherChore.java | 154 ++
 .../TableSpaceQuotaViolationNotifier.java   |  55 +
 .../hbase/regionserver/HRegionServer.java   |  21 +-
 .../hbase/regionserver/RSRpcServices.java   |   7 +-
 .../regionserver/RegionServerServices.java  |  12 +-
 .../hadoop/hbase/MockRegionServerServices.java  |  10 +-
 .../hadoop/hbase/master/MockRegionServer.java   |  10 +-
 .../TestQuotaObserverChoreWithMiniCluster.java  |   2 +
 .../hadoop/hbase/quotas/TestQuotaTableUtil.java |  47 +
 .../hadoop/hbase/quotas/TestQuotaThrottle.java  |   4 +-
 .../TestRegionServerSpaceQuotaManager.java  | 127 
 ...SpaceQuotaViolationPolicyRefresherChore.java | 131 
 .../TestTableSpaceQuotaViolationNotifier.java   | 144 +
 22 files changed, 1281 insertions(+), 226 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/dccfc846/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 8ef4f08..b5eac48 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -24,16 +24,20 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
@@ -44,7 +48,12 @@ import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.filter.RegexStringComparator;
 import org.apache.hadoop.hbase.filter.RowFilter;
 import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
+import 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Strings;
 
@@ -53,9 +62,8 @@ import org.apache.hadoop.hbase.util.Strings;
  * 
  * ROW-KEY  FAM/QUALDATA
  *   n.namespace q:s global-quotas
- *   n.namespace u:dusize in bytes
  *   t.table q:s global-quotas
- *   t.table u:dusize in bytes
+ *   t.table u:vspace violation policy
  *   u.user  q:s global-quotas
  

[12/50] [abbrv] hbase git commit: HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/c2c2178b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index d56d6ec..095f4bd 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -116,6 +116,7 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterAllFilter;
 import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.NullComparator;
@@ -4931,6 +4932,7 @@ public class TestHRegion {
   String callingMethod, Configuration conf, boolean isReadOnly, byte[]... 
families)
   throws IOException {
 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
+ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 
0, null);
 HRegionInfo hri = new HRegionInfo(tableName, startKey, stopKey);
 final WAL wal = HBaseTestingUtility.createWal(conf, logDir, hri);
 return initHRegion(tableName, startKey, stopKey, isReadOnly,

http://git-wip-us.apache.org/repos/asf/hbase/blob/c2c2178b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 0054642..6eed7df 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -153,7 +153,7 @@ public class TestHRegionReplayEvents {
 }
 
 time = System.currentTimeMillis();
-
+ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 
0, null);
 primaryHri = new HRegionInfo(htd.getTableName(),
   HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
   false, time, 0);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c2c2178b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
index 37a7664..1768801 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
@@ -48,30 +48,30 @@ import static org.junit.Assert.assertTrue;
 @Category({RegionServerTests.class, SmallTests.class})
 public class TestMemStoreChunkPool {
   private final static Configuration conf = new Configuration();
-  private static MemStoreChunkPool chunkPool;
+  private static ChunkCreator chunkCreator;
   private static boolean chunkPoolDisabledBeforeTest;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
 conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
 conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
-chunkPoolDisabledBeforeTest = MemStoreChunkPool.chunkPoolDisabled;
-MemStoreChunkPool.chunkPoolDisabled = false;
+chunkPoolDisabledBeforeTest = ChunkCreator.chunkPoolDisabled;
+ChunkCreator.chunkPoolDisabled = false;
 long globalMemStoreLimit = (long) 
(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
 .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false));
-chunkPool = MemStoreChunkPool.initialize(globalMemStoreLimit, 0.2f,
-MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, 
MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false);
-assertTrue(chunkPool != null);
+chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 
false,
+  globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
+assertTrue(chunkCreator != null);
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-MemStoreChunkPool.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
+ChunkCreator.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
   }
 
   @Before
   public void tearDown() throws Exception {
-chunkPool.clearChunks();
+chunkCreator.clearChunksInPool();
   }
 
   @Test

[49/50] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI additions for space quotas

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/96c6b8fa/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index d56def5..4577bcf 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -13024,6 +13024,3031 @@ public final class QuotaProtos {
 
   }
 
+  public interface GetQuotaStatesRequestOrBuilder extends
+  // 
@@protoc_insertion_point(interface_extends:hbase.pb.GetQuotaStatesRequest)
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+  }
+  /**
+   * Protobuf type {@code hbase.pb.GetQuotaStatesRequest}
+   */
+  public  static final class GetQuotaStatesRequest extends
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3 
implements
+  // 
@@protoc_insertion_point(message_implements:hbase.pb.GetQuotaStatesRequest)
+  GetQuotaStatesRequestOrBuilder {
+// Use GetQuotaStatesRequest.newBuilder() to construct.
+private 
GetQuotaStatesRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
 builder) {
+  super(builder);
+}
+private GetQuotaStatesRequest() {
+}
+
+@java.lang.Override
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private GetQuotaStatesRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream 
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite 
extensionRegistry)
+throws 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 {
+  this();
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder 
unknownFields =
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+  }
+}
+  } catch 
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_descriptor;
+}
+
+protected 
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.class,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+  byte isInitialized = memoizedIsInitialized;
+  if (isInitialized == 1) return true;
+  if (isInitialized == 0) return false;
+
+  memoizedIsInitialized = 1;
+  return true;
+}
+
+public void 
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream 
output)
+throws java.io.IOException {
+  unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+  int size = memoizedSize;
+  if (size != -1) return size;
+
+  size = 0;
+  size += unknownFields.getSerializedSize();
+  memoizedSize = size;
+  return size;
+}
+
+private static final long serialVersionUID = 0L;
+@java.lang.Override
+public boolean equals(final java.lang.Object obj) {
+  if (obj == this) {
+   return true;
+ 

[31/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 8b127d9..973ac8c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -37,9 +37,8 @@ import 
org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -54,51 +53,51 @@ import com.google.common.collect.Multimap;
 @InterfaceAudience.Private
 public class QuotaObserverChore extends ScheduledChore {
   private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
-  static final String VIOLATION_OBSERVER_CHORE_PERIOD_KEY =
-  "hbase.master.quotas.violation.observer.chore.period";
-  static final int VIOLATION_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 
5 minutes in millis
+  static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
+  "hbase.master.quotas.observer.chore.period";
+  static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5 
minutes in millis
 
-  static final String VIOLATION_OBSERVER_CHORE_DELAY_KEY =
-  "hbase.master.quotas.violation.observer.chore.delay";
-  static final long VIOLATION_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
+  static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
+  "hbase.master.quotas.observer.chore.delay";
+  static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
 
-  static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_KEY =
-  "hbase.master.quotas.violation.observer.chore.timeunit";
-  static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_DEFAULT = 
TimeUnit.MILLISECONDS.name();
+  static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
+  "hbase.master.quotas.observer.chore.timeunit";
+  static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT = 
TimeUnit.MILLISECONDS.name();
 
-  static final String VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY =
-  "hbase.master.quotas.violation.observer.report.percent";
-  static final double VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
+  static final String QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY =
+  "hbase.master.quotas.observer.report.percent";
+  static final double QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
 
   private final Connection conn;
   private final Configuration conf;
   private final MasterQuotaManager quotaManager;
   /*
-   * Callback that changes in quota violation are passed to.
+   * Callback that changes in quota snapshots are passed to.
*/
-  private final SpaceQuotaViolationNotifier violationNotifier;
+  private final SpaceQuotaSnapshotNotifier snapshotNotifier;
 
   /*
-   * Preserves the state of quota violations for tables and namespaces
+   * Preserves the state of quota snapshots for tables and namespaces
*/
-  private final Map tableQuotaViolationStates;
-  private final Map namespaceQuotaViolationStates;
+  private final Map tableQuotaSnapshots;
+  private final Map namespaceQuotaSnapshots;
 
   /*
-   * Encapsulates logic for moving tables/namespaces into or out of quota 
violation
+   * Encapsulates logic for tracking the state of a table/namespace WRT space 
quotas
*/
-  private QuotaViolationStore tableViolationStore;
-  private QuotaViolationStore namespaceViolationStore;
+  private QuotaSnapshotStore tableSnapshotStore;
+  private QuotaSnapshotStore namespaceSnapshotStore;
 
   public QuotaObserverChore(HMaster master) {
 this(
 master.getConnection(), master.getConfiguration(),
-master.getSpaceQuotaViolationNotifier(), 
master.getMasterQuotaManager(),
+master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
 master);
   }
 
   QuotaObserverChore(
-  Connection conn, Configuration conf, SpaceQuotaViolationNotifier 
violationNotifier,
+  Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier 
snapshotNotifier,
   

[25/50] [abbrv] hbase git commit: HBASE-16995 Build client Java API and client protobuf messages - addendum fixes white spaces (Josh Elser)

[elserj]

HBASE-16995 Build client Java API and client protobuf messages - addendum fixes 
white spaces (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eaeef44e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eaeef44e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eaeef44e

Branch: refs/heads/HBASE-16961
Commit: eaeef44e2fbd4f13a7f2d8dc5934eda3c54c529f
Parents: 990062a
Author: tedyu 
Authored: Thu Nov 17 10:42:18 2016 -0800
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:31 2017 -0400

--
 .../hbase/quotas/TestQuotaSettingsFactory.java|  2 +-
 .../shaded/protobuf/generated/MasterProtos.java   |  2 +-
 .../shaded/protobuf/generated/QuotaProtos.java| 18 +-
 .../hbase/protobuf/generated/QuotaProtos.java |  4 ++--
 4 files changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/eaeef44e/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
--
diff --git 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
index 17015d6..e0012a7 100644
--- 
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
+++ 
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
@@ -44,7 +44,7 @@ import org.junit.experimental.categories.Category;
  */
 @Category(SmallTests.class)
 public class TestQuotaSettingsFactory {
-  
+
   @Test
   public void testAllQuotasAddedToList() {
 final SpaceQuota spaceQuota = SpaceQuota.newBuilder()

http://git-wip-us.apache.org/repos/asf/hbase/blob/eaeef44e/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index 0c3248c..bbc6d1d 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -63752,7 +63752,7 @@ public final class MasterProtos {
* optional .hbase.pb.SpaceLimitRequest space_limit = 8;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
 
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
   getSpaceLimitFieldBuilder() {
 if (spaceLimitBuilder_ == null) {
   spaceLimitBuilder_ = new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<

http://git-wip-us.apache.org/repos/asf/hbase/blob/eaeef44e/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index e3c6bfd..0ab2576 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
   private 
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
 
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota, 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
 

[08/50] [abbrv] hbase git commit: HBASE-17903 Corrected the alias for the link of HBASE-6580

[elserj]

HBASE-17903 Corrected the alias for the link of HBASE-6580

Signed-off-by: CHIA-PING TSAI 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/918aa465
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/918aa465
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/918aa465

Branch: refs/heads/HBASE-16961
Commit: 918aa4655c4109159f27b6d78460bd3681c11f06
Parents: 8db9760
Author: Jan Hentschel 
Authored: Sun Apr 16 17:02:47 2017 +0200
Committer: CHIA-PING TSAI 
Committed: Mon Apr 17 10:22:25 2017 +0800

--
 src/main/asciidoc/_chapters/architecture.adoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/918aa465/src/main/asciidoc/_chapters/architecture.adoc
--
diff --git a/src/main/asciidoc/_chapters/architecture.adoc 
b/src/main/asciidoc/_chapters/architecture.adoc
index 773d237..27aebd9 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -230,7 +230,7 @@ try (Connection connection = 
ConnectionFactory.createConnection(conf)) {
 .`HTablePool` is Deprecated
 [WARNING]
 
-Previous versions of this guide discussed `HTablePool`, which was deprecated 
in HBase 0.94, 0.95, and 0.96, and removed in 0.98.1, by 
link:https://issues.apache.org/jira/browse/HBASE-6580[HBASE-6500], or 
`HConnection`, which is deprecated in HBase 1.0 by `Connection`.
+Previous versions of this guide discussed `HTablePool`, which was deprecated 
in HBase 0.94, 0.95, and 0.96, and removed in 0.98.1, by 
link:https://issues.apache.org/jira/browse/HBASE-6580[HBASE-6580], or 
`HConnection`, which is deprecated in HBase 1.0 by `Connection`.
 Please use 
link:http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Connection.html[Connection]
 instead.
 
 

[17/50] [abbrv] hbase git commit: HBASE-16998 Implement Master-side analysis of region space reports

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/bcf6da40/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 000..98236c2
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+  private static final Log LOG = 
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+  private static final int SIZE_PER_VALUE = 256;
+  private static final String F1 = "f1";
+  private static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+  private static final AtomicLong COUNTER = new AtomicLong(0);
+  private static final long ONE_MEGABYTE = 1024L * 1024L;
+  private static final long DEFAULT_WAIT_MILLIS = 500;
+
+  @Rule
+  public TestName testName = new TestName();
+
+  private HMaster master;
+  private QuotaObserverChore chore;
+  private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 
1000);
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 
1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void removeAllQuotas() throws Exception {
+final Connection conn = TEST_UTIL.getConnection();
+

[05/50] [abbrv] hbase git commit: Revert "HBASE-17906 When a huge amount of data writing to hbase through thrift2, there will be a deadlock error. (Albert Lee)" Mistaken commit.

[elserj]

Revert "HBASE-17906 When a huge amount of data writing to hbase through 
thrift2, there will be a deadlock error. (Albert Lee)"
Mistaken commit.

This reverts commit 9dd5cda01747ffb91ac084792fa4a8670859e810.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0cd4cec5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0cd4cec5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0cd4cec5

Branch: refs/heads/HBASE-16961
Commit: 0cd4cec5d24b5e7194a903e4d900f5558ed8b9a7
Parents: c846145
Author: Michael Stack 
Authored: Fri Apr 14 12:07:40 2017 -0700
Committer: Michael Stack 
Committed: Fri Apr 14 12:07:40 2017 -0700

--
 .../main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0cd4cec5/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
--
diff --git 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index 8f56b10..560ae64 100644
--- 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++ 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -432,6 +432,9 @@ public class ThriftServer extends Configured implements 
Tool {
   throw new RuntimeException("Could not parse the value provided for the 
port option", e);
 }
 
+// Thrift's implementation uses '0' as a placeholder for 'use the default.'
+int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
+
 // Local hostname and user name,
 // used only if QOP is configured.
 String host = null;

[06/50] [abbrv] hbase git commit: HBASE-17904 Get runs into NoSuchElementException when using Read Replica, with hbase. ipc.client.specificThreadForWriting to be true and hbase.rpc.client.impl to be o

[elserj]

HBASE-17904 Get runs into NoSuchElementException when using Read Replica, with 
hbase. ipc.client.specificThreadForWriting
to be true and hbase.rpc.client.impl to be 
org.apache.hadoop.hbase.ipc.RpcClientImpl (Huaxiang Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7678855f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7678855f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7678855f

Branch: refs/heads/HBASE-16961
Commit: 7678855fac011a9c02e5d6a42470c0178482a4ce
Parents: 0cd4cec
Author: Michael Stack 
Authored: Sun Apr 16 11:00:57 2017 -0700
Committer: Michael Stack 
Committed: Sun Apr 16 11:01:06 2017 -0700

--
 .../hadoop/hbase/ipc/BlockingRpcConnection.java |  2 +-
 .../hbase/client/TestReplicaWithCluster.java| 50 
 2 files changed, 51 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7678855f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
index 15eb10c..1012ad0 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java
@@ -156,7 +156,7 @@ class BlockingRpcConnection extends RpcConnection 
implements Runnable {
 }
 
 public void remove(Call call) {
-  callsToWrite.remove();
+  callsToWrite.remove(call);
   // By removing the call from the expected call list, we make the list 
smaller, but
   // it means as well that we don't know how many calls we cancelled.
   calls.remove(call.id);

http://git-wip-us.apache.org/repos/asf/hbase/blob/7678855f/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
index becb2eb..2c77541 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicaWithCluster.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.Waiter;
+
 import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
@@ -515,7 +516,56 @@ public class TestReplicaWithCluster {
 
   Assert.assertTrue(r.isStale());
 } finally {
+  HTU.getAdmin().disableTable(hdt.getTableName());
+  HTU.deleteTable(hdt.getTableName());
+}
+  }
+
+  @Test
+  public void testReplicaGetWithRpcClientImpl() throws IOException {
+
HTU.getConfiguration().setBoolean("hbase.ipc.client.specificThreadForWriting", 
true);
+HTU.getConfiguration().set("hbase.rpc.client.impl", 
"org.apache.hadoop.hbase.ipc.RpcClientImpl");
+// Create table then get the single region for our new table.
+HTableDescriptor hdt = 
HTU.createTableDescriptor("testReplicaGetWithRpcClientImpl");
+hdt.setRegionReplication(NB_SERVERS);
+hdt.addCoprocessor(SlowMeCopro.class.getName());
+
+try {
+  Table table = HTU.createTable(hdt, new byte[][] { f }, null);
+
+  Put p = new Put(row);
+  p.addColumn(f, row, row);
+  table.put(p);
 
+  // Flush so it can be picked by the replica refresher thread
+  HTU.flush(table.getName());
+
+  // Sleep for some time until data is picked up by replicas
+  try {
+Thread.sleep(2 * REFRESH_PERIOD);
+  } catch (InterruptedException e1) {
+LOG.error(e1);
+  }
+
+  try {
+// Create the new connection so new config can kick in
+Connection connection = 
ConnectionFactory.createConnection(HTU.getConfiguration());
+Table t = connection.getTable(hdt.getTableName());
+
+// But if we ask for stale we will get it
+SlowMeCopro.cdl.set(new CountDownLatch(1));
+Get g = new Get(row);
+g.setConsistency(Consistency.TIMELINE);
+Result r = t.get(g);
+Assert.assertTrue(r.isStale());
+SlowMeCopro.cdl.get().countDown();
+  } finally {
+SlowMeCopro.cdl.get().countDown();
+SlowMeCopro.sleepTime.set(0);
+  }
+} 

[39/50] [abbrv] hbase git commit: HBASE-17602 Reduce some quota chore periods/delays

[elserj]

HBASE-17602 Reduce some quota chore periods/delays


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4db44ad6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4db44ad6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4db44ad6

Branch: refs/heads/HBASE-16961
Commit: 4db44ad66497729118b695e22fd2e0f4c7787cc9
Parents: 48332ee
Author: Josh Elser 
Authored: Tue Feb 7 11:21:08 2017 -0500
Committer: Josh Elser 
Committed: Mon Apr 17 15:44:00 2017 -0400

--
 .../java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java  | 4 ++--
 .../org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4db44ad6/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index b9f4592..7f894e4 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -55,11 +55,11 @@ public class QuotaObserverChore extends ScheduledChore {
   private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
   static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
   "hbase.master.quotas.observer.chore.period";
-  static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5 
minutes in millis
+  static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1 
minutes in millis
 
   static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
   "hbase.master.quotas.observer.chore.delay";
-  static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
+  static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15 
seconds in millis
 
   static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
   "hbase.master.quotas.observer.chore.timeunit";

http://git-wip-us.apache.org/repos/asf/hbase/blob/4db44ad6/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
index e1a2693..8587e79 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
@@ -44,11 +44,11 @@ public class SpaceQuotaRefresherChore extends 
ScheduledChore {
 
   static final String POLICY_REFRESHER_CHORE_PERIOD_KEY =
   "hbase.regionserver.quotas.policy.refresher.chore.period";
-  static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5 
minutes in millis
+  static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1 
minute in millis
 
   static final String POLICY_REFRESHER_CHORE_DELAY_KEY =
   "hbase.regionserver.quotas.policy.refresher.chore.delay";
-  static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1 
minute
+  static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15 
seconds in millis
 
   static final String POLICY_REFRESHER_CHORE_TIMEUNIT_KEY =
   "hbase.regionserver.quotas.policy.refresher.chore.timeunit";

[27/50] [abbrv] hbase git commit: HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota table (Josh Elser)

[elserj]

HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota 
table (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a29abe64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a29abe64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a29abe64

Branch: refs/heads/HBASE-16961
Commit: a29abe646e2b1cd75ee1d0f186cc7486fe78b79e
Parents: 988a23e
Author: tedyu 
Authored: Sat Dec 3 14:30:48 2016 -0800
Committer: Josh Elser 
Committed: Mon Apr 17 15:35:31 2017 -0400

--
 .../hadoop/hbase/quotas/QuotaTableUtil.java |  13 +-
 .../hadoop/hbase/quotas/MasterQuotaManager.java |  30 +
 .../hadoop/hbase/quotas/TestQuotaAdmin.java | 125 ++-
 3 files changed, 165 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a29abe64/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index c44090f..8ef4f08 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -53,7 +53,9 @@ import org.apache.hadoop.hbase.util.Strings;
  * 
  * ROW-KEY  FAM/QUALDATA
  *   n.namespace q:s global-quotas
+ *   n.namespace u:dusize in bytes
  *   t.table q:s global-quotas
+ *   t.table u:dusize in bytes
  *   u.user  q:s global-quotas
  *   u.user  q:s.table table-quotas
  *   u.user  q:s.ns:   namespace-quotas
@@ -72,6 +74,7 @@ public class QuotaTableUtil {
   protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
   protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
   protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX = 
Bytes.toBytes("s.");
+  protected static final byte[] QUOTA_QUALIFIER_DISKUSAGE = 
Bytes.toBytes("du");
   protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = 
Bytes.toBytes("u.");
   protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = 
Bytes.toBytes("t.");
   protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = 
Bytes.toBytes("n.");
@@ -330,11 +333,16 @@ public class QuotaTableUtil {
*  Quotas protobuf helpers
*/
   protected static Quotas quotasFromData(final byte[] data) throws IOException 
{
+return quotasFromData(data, 0, data.length);
+  }
+
+  protected static Quotas quotasFromData(
+  final byte[] data, int offset, int length) throws IOException {
 int magicLen = ProtobufMagic.lengthOfPBMagic();
-if (!ProtobufMagic.isPBMagicPrefix(data, 0, magicLen)) {
+if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) {
   throw new IOException("Missing pb magic prefix");
 }
-return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen, 
data.length - magicLen));
+return Quotas.parseFrom(new ByteArrayInputStream(data, offset + magicLen, 
length - magicLen));
   }
 
   protected static byte[] quotasToData(final Quotas data) throws IOException {
@@ -348,6 +356,7 @@ public class QuotaTableUtil {
 boolean hasSettings = false;
 hasSettings |= quotas.hasThrottle();
 hasSettings |= quotas.hasBypassGlobals();
+hasSettings |= quotas.hasSpace();
 return !hasSettings;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/a29abe64/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 5dab2e3..1469268 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
 import 

[13/50] [abbrv] hbase git commit: HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)

[elserj]

HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c2c2178b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c2c2178b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c2c2178b

Branch: refs/heads/HBASE-16961
Commit: c2c2178b2eebe4439eadec6b37fae2566944c16b
Parents: c8cd921
Author: Ramkrishna 
Authored: Mon Apr 17 09:10:59 2017 +0530
Committer: Ramkrishna 
Committed: Mon Apr 17 09:28:24 2017 +0530

--
 .../java/org/apache/hadoop/hbase/CellUtil.java  |  24 --
 .../org/apache/hadoop/hbase/ExtendedCell.java   |  10 +
 .../org/apache/hadoop/hbase/master/HMaster.java |   2 +
 .../hbase/regionserver/ByteBufferChunkCell.java |  48 +++
 .../apache/hadoop/hbase/regionserver/Chunk.java |  60 ++-
 .../hadoop/hbase/regionserver/ChunkCreator.java | 404 +++
 .../hbase/regionserver/HRegionServer.java   |  14 +-
 .../hbase/regionserver/MemStoreChunkPool.java   | 265 
 .../hadoop/hbase/regionserver/MemStoreLAB.java  |   4 +-
 .../hbase/regionserver/MemStoreLABImpl.java | 171 
 .../regionserver/NoTagByteBufferChunkCell.java  |  48 +++
 .../hadoop/hbase/regionserver/OffheapChunk.java |  31 +-
 .../hadoop/hbase/regionserver/OnheapChunk.java  |  32 +-
 .../hadoop/hbase/HBaseTestingUtility.java   |   3 +
 .../coprocessor/TestCoprocessorInterface.java   |   4 +
 .../TestRegionObserverScannerOpenHook.java  |   3 +
 .../coprocessor/TestRegionObserverStacking.java |   3 +
 .../io/hfile/TestScannerFromBucketCache.java|   3 +
 .../hadoop/hbase/master/TestCatalogJanitor.java |   7 +
 .../hadoop/hbase/regionserver/TestBulkLoad.java |   2 +-
 .../hbase/regionserver/TestCellFlatSet.java |   2 +-
 .../regionserver/TestCompactingMemStore.java|  37 +-
 .../TestCompactingToCellArrayMapMemStore.java   |  16 +-
 .../TestCompactionArchiveConcurrentClose.java   |   1 +
 .../TestCompactionArchiveIOException.java   |   1 +
 .../regionserver/TestCompactionPolicy.java  |   1 +
 .../hbase/regionserver/TestDefaultMemStore.java |  14 +-
 .../regionserver/TestFailedAppendAndSync.java   |   1 +
 .../hbase/regionserver/TestHMobStore.java   |   2 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |   2 +
 .../regionserver/TestHRegionReplayEvents.java   |   2 +-
 .../regionserver/TestMemStoreChunkPool.java |  48 +--
 .../hbase/regionserver/TestMemStoreLAB.java |  27 +-
 .../TestMemstoreLABWithoutPool.java | 168 
 .../hbase/regionserver/TestRecoveredEdits.java  |   1 +
 .../hbase/regionserver/TestRegionIncrement.java |   1 +
 .../hadoop/hbase/regionserver/TestStore.java|   1 +
 .../TestStoreFileRefresherChore.java|   1 +
 .../hbase/regionserver/TestWALLockup.java   |   1 +
 .../TestWALMonotonicallyIncreasingSeqId.java|   1 +
 .../hbase/regionserver/wal/TestDurability.java  |   3 +
 41 files changed, 990 insertions(+), 479 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c2c2178b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index e1bc969..56de21b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -3135,28 +3135,4 @@ public final class CellUtil {
   return Type.DeleteFamily.getCode();
 }
   }
-
-  /**
-   * Clone the passed cell by copying its data into the passed buf.
-   */
-  public static Cell copyCellTo(Cell cell, ByteBuffer buf, int offset, int 
len) {
-int tagsLen = cell.getTagsLength();
-if (cell instanceof ExtendedCell) {
-  ((ExtendedCell) cell).write(buf, offset);
-} else {
-  // Normally all Cell impls within Server will be of type ExtendedCell. 
Just considering the
-  // other case also. The data fragments within Cell is copied into buf as 
in KeyValue
-  // serialization format only.
-  KeyValueUtil.appendTo(cell, buf, offset, true);
-}
-if (tagsLen == 0) {
-  // When tagsLen is 0, make a NoTagsByteBufferKeyValue version. This is 
an optimized class
-  // which directly return tagsLen as 0. So we avoid parsing many length 
components in
-  // reading the tagLength stored in the backing buffer. The Memstore 
addition of every Cell
-  // call getTagsLength().
-  return new NoTagsByteBufferKeyValue(buf, offset, len, 
cell.getSequenceId());
-} else {
-  return new ByteBufferKeyValue(buf, offset, len, cell.getSequenceId());
-}
-  }
 }

[47/50] [abbrv] hbase git commit: HBASE-17794 Swap "violation" for "snapshot" where appropriate

[elserj]

HBASE-17794 Swap "violation" for "snapshot" where appropriate

A couple of variables and comments in which violation is incorrectly
used to describe what the code is doing. This was a hold over from early
implementation -- need to scrub these out for clarity.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/abe1c065
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/abe1c065
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/abe1c065

Branch: refs/heads/HBASE-16961
Commit: abe1c065ab803ba60fa28b17432746e756c31ed7
Parents: 5b3926b
Author: Josh Elser 
Authored: Thu Mar 16 19:26:14 2017 -0400
Committer: Josh Elser 
Committed: Mon Apr 17 15:47:49 2017 -0400

--
 .../java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java| 4 ++--
 hbase-protocol-shaded/src/main/protobuf/Quota.proto| 2 +-
 .../org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 6 +++---
 .../apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java| 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index ad59517..c008702 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -228,7 +228,7 @@ public class QuotaTableUtil {
   }
 
   /**
-   * Creates a {@link Scan} which returns only quota violations from the quota 
table.
+   * Creates a {@link Scan} which returns only quota snapshots from the quota 
table.
*/
   public static Scan makeQuotaSnapshotScan() {
 Scan s = new Scan();
@@ -246,7 +246,7 @@ public class QuotaTableUtil {
* will throw an {@link IllegalArgumentException}.
*
* @param result A row from the quota table.
-   * @param snapshots A map of violations to add the result of this method 
into.
+   * @param snapshots A map of snapshots to add the result of this method into.
*/
   public static void extractQuotaSnapshot(
   Result result, Map snapshots) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto 
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 1a6d5ed..364c58b 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -98,7 +98,7 @@ message SpaceLimitRequest {
 }
 
 // Represents the state of a quota on a table. Either the quota is not in 
violation
-// or it is in violatino there is a violation policy which should be in effect.
+// or it is in violation there is a violation policy which should be in effect.
 message SpaceQuotaStatus {
   optional SpaceViolationPolicy policy = 1;
   optional bool in_violation = 2;

http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 94c5c87..254f2a1 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -532,9 +532,9 @@ public class QuotaObserverChore extends ScheduledChore {
   }
 
   /**
-   * Stores the quota violation state for the given table.
+   * Stores the quota state for the given table.
*/
-  void setTableQuotaViolation(TableName table, SpaceQuotaSnapshot snapshot) {
+  void setTableQuotaSnapshot(TableName table, SpaceQuotaSnapshot snapshot) {
 this.tableQuotaSnapshots.put(table, snapshot);
   }
 
@@ -552,7 +552,7 @@ public class QuotaObserverChore extends ScheduledChore {
   }
 
   /**
-   * Stores the quota violation state for the given namespace.
+   * Stores the quota state for the given namespace.
*/
   void setNamespaceQuotaSnapshot(String namespace, SpaceQuotaSnapshot 
snapshot) {
 this.namespaceQuotaSnapshots.put(namespace, snapshot);

[14/50] [abbrv] hbase git commit: Revert "HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)"

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 095f4bd..d56d6ec 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -116,7 +116,6 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterAllFilter;
 import org.apache.hadoop.hbase.filter.FilterBase;
 import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.NullComparator;
@@ -4932,7 +4931,6 @@ public class TestHRegion {
   String callingMethod, Configuration conf, boolean isReadOnly, byte[]... 
families)
   throws IOException {
 Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
-ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 
0, null);
 HRegionInfo hri = new HRegionInfo(tableName, startKey, stopKey);
 final WAL wal = HBaseTestingUtility.createWal(conf, logDir, hri);
 return initHRegion(tableName, startKey, stopKey, isReadOnly,

http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 6eed7df..0054642 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -153,7 +153,7 @@ public class TestHRegionReplayEvents {
 }
 
 time = System.currentTimeMillis();
-ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 
0, null);
+
 primaryHri = new HRegionInfo(htd.getTableName(),
   HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
   false, time, 0);

http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
index 1768801..37a7664 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
@@ -48,30 +48,30 @@ import static org.junit.Assert.assertTrue;
 @Category({RegionServerTests.class, SmallTests.class})
 public class TestMemStoreChunkPool {
   private final static Configuration conf = new Configuration();
-  private static ChunkCreator chunkCreator;
+  private static MemStoreChunkPool chunkPool;
   private static boolean chunkPoolDisabledBeforeTest;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
 conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
 conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
-chunkPoolDisabledBeforeTest = ChunkCreator.chunkPoolDisabled;
-ChunkCreator.chunkPoolDisabled = false;
+chunkPoolDisabledBeforeTest = MemStoreChunkPool.chunkPoolDisabled;
+MemStoreChunkPool.chunkPoolDisabled = false;
 long globalMemStoreLimit = (long) 
(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
 .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false));
-chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 
false,
-  globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
-assertTrue(chunkCreator != null);
+chunkPool = MemStoreChunkPool.initialize(globalMemStoreLimit, 0.2f,
+MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, 
MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false);
+assertTrue(chunkPool != null);
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-ChunkCreator.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
+MemStoreChunkPool.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
   }
 
   @Before
   public void tearDown() throws Exception {
-chunkCreator.clearChunksInPool();
+chunkPool.clearChunks();
   }
 
   @Test

[20/50] [abbrv] hbase git commit: HBASE-17000 Implement computation of online region sizes and report to the Master

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index d7d4db0..e90c934 100644
--- 
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++ 
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10164,6 +10164,1912 @@ public final class RegionServerStatusProtos {
 
   }
 
+  public interface RegionSpaceUseOrBuilder extends
+  // @@protoc_insertion_point(interface_extends:hbase.pb.RegionSpaceUse)
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ * 
+ * A region identifier
+ * 
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+boolean hasRegion();
+/**
+ * 
+ * A region identifier
+ * 
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo 
getRegion();
+/**
+ * 
+ * A region identifier
+ * 
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
 getRegionOrBuilder();
+
+/**
+ * 
+ * The size in bytes of the region
+ * 
+ *
+ * optional uint64 size = 2;
+ */
+boolean hasSize();
+/**
+ * 
+ * The size in bytes of the region
+ * 
+ *
+ * optional uint64 size = 2;
+ */
+long getSize();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.RegionSpaceUse}
+   */
+  public  static final class RegionSpaceUse extends
+  org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3 
implements
+  // @@protoc_insertion_point(message_implements:hbase.pb.RegionSpaceUse)
+  RegionSpaceUseOrBuilder {
+// Use RegionSpaceUse.newBuilder() to construct.
+private 
RegionSpaceUse(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
 builder) {
+  super(builder);
+}
+private RegionSpaceUse() {
+  size_ = 0L;
+}
+
+@java.lang.Override
+public final 
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private RegionSpaceUse(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream 
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite 
extensionRegistry)
+throws 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 {
+  this();
+  int mutable_bitField0_ = 0;
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder 
unknownFields =
+  
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 10: {
+  
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
 subBuilder = null;
+  if (((bitField0_ & 0x0001) == 0x0001)) {
+subBuilder = region_.toBuilder();
+  }
+  region_ = 
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
 extensionRegistry);
+  if (subBuilder != null) {
+subBuilder.mergeFrom(region_);
+region_ = subBuilder.buildPartial();
+  }
+  bitField0_ |= 0x0001;
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  size_ = input.readUInt64();
+  break;
+}
+  }
+}
+  } catch 
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
 e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new 
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+

[19/50] [abbrv] hbase git commit: HBASE-17000 Implement computation of online region sizes and report to the Master

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto 
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 1c373ee..23ddd43 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -141,6 +141,22 @@ message SplitTableRegionResponse {
   optional uint64 proc_id = 1;
 }
 
+message RegionSpaceUse {
+  optional RegionInfo region = 1; // A region identifier
+  optional uint64 size = 2; // The size in bytes of the region
+}
+
+/**
+ * Reports filesystem usage for regions.
+ */
+message RegionSpaceUseReportRequest {
+  repeated RegionSpaceUse space_use = 1;
+}
+
+message RegionSpaceUseReportResponse {
+
+}
+
 service RegionServerStatusService {
   /** Called when a region server first starts. */
   rpc RegionServerStartup(RegionServerStartupRequest)
@@ -182,4 +198,10 @@ service RegionServerStatusService {
*/
   rpc getProcedureResult(GetProcedureResultRequest)
 returns(GetProcedureResultResponse);
+
+  /**
+   * Reports Region filesystem space use
+   */
+  rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
+returns(RegionSpaceUseReportResponse);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 9af8f45..f454248 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
 import org.apache.hadoop.hbase.replication.ReplicationException;
 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -216,6 +217,9 @@ import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
+import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
 import 
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
@@ -2006,4 +2010,19 @@ public class MasterRpcServices extends RSRpcServices
   throw new ServiceException(e);
 }
   }
+
+  @Override
+  public RegionSpaceUseReportResponse reportRegionSpaceUse(RpcController 
controller,
+  RegionSpaceUseReportRequest request) throws ServiceException {
+try {
+  master.checkInitialized();
+  MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+  for (RegionSpaceUse report : request.getSpaceUseList()) {
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()), 
report.getSize());
+  }
+  return RegionSpaceUseReportResponse.newBuilder().build();
+} catch (Exception e) {
+  throw new ServiceException(e);
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
new file mode 100644
index 000..01540eb
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or 

[32/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer

[elserj]

http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index cc40536..d466e59 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -5778,6 +5778,1284 @@ public final class QuotaProtos {
 // @@protoc_insertion_point(class_scope:hbase.pb.SpaceLimitRequest)
   }
 
+  public interface SpaceQuotaStatusOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// optional .hbase.pb.SpaceViolationPolicy policy = 1;
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+boolean hasPolicy();
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy 
getPolicy();
+
+// optional bool in_violation = 2;
+/**
+ * optional bool in_violation = 2;
+ */
+boolean hasInViolation();
+/**
+ * optional bool in_violation = 2;
+ */
+boolean getInViolation();
+  }
+  /**
+   * Protobuf type {@code hbase.pb.SpaceQuotaStatus}
+   *
+   * 
+   * Represents the state of a quota on a table. Either the quota is not in 
violation
+   * or it is in violatino there is a violation policy which should be in 
effect.
+   * 
+   */
+  public static final class SpaceQuotaStatus extends
+  com.google.protobuf.GeneratedMessage
+  implements SpaceQuotaStatusOrBuilder {
+// Use SpaceQuotaStatus.newBuilder() to construct.
+private SpaceQuotaStatus(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private SpaceQuotaStatus(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SpaceQuotaStatus defaultInstance;
+public static SpaceQuotaStatus getDefaultInstance() {
+  return defaultInstance;
+}
+
+public SpaceQuotaStatus getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private SpaceQuotaStatus(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 8: {
+  int rawValue = input.readEnum();
+  
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy 
value = 
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy.valueOf(rawValue);
+  if (value == null) {
+unknownFields.mergeVarintField(1, rawValue);
+  } else {
+bitField0_ |= 0x0001;
+policy_ = value;
+  }
+  break;
+}
+case 16: {
+  bitField0_ |= 0x0002;
+  inViolation_ = input.readBool();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.internal_static_hbase_pb_SpaceQuotaStatus_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return