hbase git commit: HBASE-16303 FilterList with MUST_PASS_ONE optimization (Ram)

2016-08-04 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 f49d2fc7e -> 8570e7f8d


HBASE-16303 FilterList with MUST_PASS_ONE optimization (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8570e7f8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8570e7f8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8570e7f8

Branch: refs/heads/branch-1.3
Commit: 8570e7f8dbfb42bcb686953fbb4eadac860eeb21
Parents: f49d2fc
Author: Ramkrishna 
Authored: Fri Aug 5 10:58:02 2016 +0530
Committer: Ramkrishna 
Committed: Fri Aug 5 11:03:05 2016 +0530

--
 .../test/java/org/apache/hadoop/hbase/filter/TestFilter.java| 5 +
 .../java/org/apache/hadoop/hbase/filter/TestFilterList.java | 3 +++
 2 files changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8570e7f8/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index f490b03..b8883bc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -315,6 +315,7 @@ public class TestFilter {
 verifyScan(s, expectedRows, expectedKeys);
   }
 
+  @Test
   public void testPrefixFilterWithReverseScan() throws Exception {
 // Grab rows from group one (half of total)
 long expectedRows = this.numRows / 2;
@@ -412,6 +413,7 @@ public class TestFilter {
 
   }
 
+  @Test
   public void testPageFilterWithReverseScan() throws Exception {
 // KVs in first 6 rows
 KeyValue[] expectedKVs = {
@@ -491,6 +493,7 @@ public class TestFilter {
 verifyScan(s, expectedRows, expectedKeys);
   }
 
+  @Test
   public void testWhileMatchFilterWithFilterRowWithReverseScan()
   throws Exception {
 final int pageSize = 4;
@@ -520,6 +523,7 @@ public class TestFilter {
 pageSize, scannerCounter);
   }
 
+  @Test
   public void testWhileMatchFilterWithFilterRowKeyWithReverseScan()
   throws Exception {
 Scan s = new Scan();
@@ -1767,6 +1771,7 @@ public class TestFilter {
 kvs.length, idx);
   }
 
+  @Test
   public void testColumnPaginationFilterColumnOffset() throws Exception {
 KeyValue [] expectedKVs = {
   // testRowOne-0

http://git-wip-us.apache.org/repos/asf/hbase/blob/8570e7f8/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index e719a2a..6ddc422 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -261,6 +261,7 @@ public class TestFilterList {
* we expect to get the same result as the 'prefix' only result.
* @throws Exception
*/
+  @Test
   public void testFilterListTwoFiltersMustPassOne() throws Exception {
 byte[] r1 = Bytes.toBytes("Row1");
 byte[] r11 = Bytes.toBytes("Row11");
@@ -293,6 +294,7 @@ public class TestFilterList {
* we expect to get the same result as the inclusive stop result.
* @throws Exception
*/
+  @Test
   public void testFilterListWithInclusiveStopFilteMustPassOne() throws 
Exception {
 byte[] r1 = Bytes.toBytes("Row1");
 byte[] r11 = Bytes.toBytes("Row11");
@@ -352,6 +354,7 @@ public class TestFilterList {
* Test filterKeyValue logic.
* @throws Exception
*/
+  @Test
   public void testFilterKeyValue() throws Exception {
 Filter includeFilter = new FilterBase() {
   @Override

hbase git commit: HBASE-16303 FilterList with MUST_PASS_ONE optimization (Ram)

2016-08-04 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 9c8cbd44e -> 6278ece8b


HBASE-16303 FilterList with MUST_PASS_ONE optimization (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6278ece8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6278ece8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6278ece8

Branch: refs/heads/branch-1
Commit: 6278ece8b6cbe3fae0e8618dd25ab03533113095
Parents: 9c8cbd4
Author: Ramkrishna 
Authored: Fri Aug 5 10:58:02 2016 +0530
Committer: Ramkrishna 
Committed: Fri Aug 5 10:59:19 2016 +0530

--
 .../test/java/org/apache/hadoop/hbase/filter/TestFilter.java| 5 +
 .../java/org/apache/hadoop/hbase/filter/TestFilterList.java | 3 +++
 2 files changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6278ece8/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index f490b03..b8883bc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -315,6 +315,7 @@ public class TestFilter {
 verifyScan(s, expectedRows, expectedKeys);
   }
 
+  @Test
   public void testPrefixFilterWithReverseScan() throws Exception {
 // Grab rows from group one (half of total)
 long expectedRows = this.numRows / 2;
@@ -412,6 +413,7 @@ public class TestFilter {
 
   }
 
+  @Test
   public void testPageFilterWithReverseScan() throws Exception {
 // KVs in first 6 rows
 KeyValue[] expectedKVs = {
@@ -491,6 +493,7 @@ public class TestFilter {
 verifyScan(s, expectedRows, expectedKeys);
   }
 
+  @Test
   public void testWhileMatchFilterWithFilterRowWithReverseScan()
   throws Exception {
 final int pageSize = 4;
@@ -520,6 +523,7 @@ public class TestFilter {
 pageSize, scannerCounter);
   }
 
+  @Test
   public void testWhileMatchFilterWithFilterRowKeyWithReverseScan()
   throws Exception {
 Scan s = new Scan();
@@ -1767,6 +1771,7 @@ public class TestFilter {
 kvs.length, idx);
   }
 
+  @Test
   public void testColumnPaginationFilterColumnOffset() throws Exception {
 KeyValue [] expectedKVs = {
   // testRowOne-0

http://git-wip-us.apache.org/repos/asf/hbase/blob/6278ece8/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index e719a2a..6ddc422 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -261,6 +261,7 @@ public class TestFilterList {
* we expect to get the same result as the 'prefix' only result.
* @throws Exception
*/
+  @Test
   public void testFilterListTwoFiltersMustPassOne() throws Exception {
 byte[] r1 = Bytes.toBytes("Row1");
 byte[] r11 = Bytes.toBytes("Row11");
@@ -293,6 +294,7 @@ public class TestFilterList {
* we expect to get the same result as the inclusive stop result.
* @throws Exception
*/
+  @Test
   public void testFilterListWithInclusiveStopFilteMustPassOne() throws 
Exception {
 byte[] r1 = Bytes.toBytes("Row1");
 byte[] r11 = Bytes.toBytes("Row11");
@@ -352,6 +354,7 @@ public class TestFilterList {
* Test filterKeyValue logic.
* @throws Exception
*/
+  @Test
   public void testFilterKeyValue() throws Exception {
 Filter includeFilter = new FilterBase() {
   @Override

hbase git commit: HBASE-16303 FilterList with MUST_PASS_ONE optimization (Ram)

2016-08-04 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/master 5e23b3aad -> 19c609fa5


HBASE-16303 FilterList with MUST_PASS_ONE optimization (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/19c609fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/19c609fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/19c609fa

Branch: refs/heads/master
Commit: 19c609fa5e2624e537a445d9204f06e56ae782c2
Parents: 5e23b3a
Author: Ramkrishna 
Authored: Fri Aug 5 10:58:02 2016 +0530
Committer: Ramkrishna 
Committed: Fri Aug 5 10:58:02 2016 +0530

--
 .../test/java/org/apache/hadoop/hbase/filter/TestFilter.java| 5 +
 .../java/org/apache/hadoop/hbase/filter/TestFilterList.java | 3 +++
 2 files changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/19c609fa/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
index 1dd6616..4b8da96 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java
@@ -315,6 +315,7 @@ public class TestFilter {
 verifyScan(s, expectedRows, expectedKeys);
   }
 
+  @Test
   public void testPrefixFilterWithReverseScan() throws Exception {
 // Grab rows from group one (half of total)
 long expectedRows = this.numRows / 2;
@@ -412,6 +413,7 @@ public class TestFilter {
 
   }
 
+  @Test
   public void testPageFilterWithReverseScan() throws Exception {
 // KVs in first 6 rows
 KeyValue[] expectedKVs = {
@@ -491,6 +493,7 @@ public class TestFilter {
 verifyScan(s, expectedRows, expectedKeys);
   }
 
+  @Test
   public void testWhileMatchFilterWithFilterRowWithReverseScan()
   throws Exception {
 final int pageSize = 4;
@@ -520,6 +523,7 @@ public class TestFilter {
 pageSize, scannerCounter);
   }
 
+  @Test
   public void testWhileMatchFilterWithFilterRowKeyWithReverseScan()
   throws Exception {
 Scan s = new Scan();
@@ -1766,6 +1770,7 @@ public class TestFilter {
 kvs.length, idx);
   }
 
+  @Test
   public void testColumnPaginationFilterColumnOffset() throws Exception {
 KeyValue [] expectedKVs = {
   // testRowOne-0

http://git-wip-us.apache.org/repos/asf/hbase/blob/19c609fa/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
index 440c9f5..1211e39 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterList.java
@@ -262,6 +262,7 @@ public class TestFilterList {
* we expect to get the same result as the 'prefix' only result.
* @throws Exception
*/
+  @Test
   public void testFilterListTwoFiltersMustPassOne() throws Exception {
 byte[] r1 = Bytes.toBytes("Row1");
 byte[] r11 = Bytes.toBytes("Row11");
@@ -294,6 +295,7 @@ public class TestFilterList {
* we expect to get the same result as the inclusive stop result.
* @throws Exception
*/
+  @Test
   public void testFilterListWithInclusiveStopFilteMustPassOne() throws 
Exception {
 byte[] r1 = Bytes.toBytes("Row1");
 byte[] r11 = Bytes.toBytes("Row11");
@@ -353,6 +355,7 @@ public class TestFilterList {
* Test filterKeyValue logic.
* @throws Exception
*/
+  @Test
   public void testFilterKeyValue() throws Exception {
 Filter includeFilter = new FilterBase() {
   @Override

hbase git commit: HBASE-16359 NullPointerException in RSRpcServices.openRegion()

2016-08-04 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/master c9f84e813 -> 5e23b3aad


HBASE-16359 NullPointerException in RSRpcServices.openRegion()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e23b3aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e23b3aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e23b3aa

Branch: refs/heads/master
Commit: 5e23b3aad5fffbb564d793f1d722d152e39f628a
Parents: c9f84e8
Author: tedyu 
Authored: Thu Aug 4 20:46:48 2016 -0700
Committer: tedyu 
Committed: Thu Aug 4 20:46:48 2016 -0700

--
 .../java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java  | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5e23b3aa/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index f9b78e1..0de4a28 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -1741,6 +1741,9 @@ public class RSRpcServices implements 
HBaseRPCErrorHandler,
   } else {
 
regionServer.updateRegionFavoredNodesMapping(region.getEncodedName(),
   regionOpenInfo.getFavoredNodesList());
+if (htd == null) {
+  throw new IOException("Missing table descriptor for " + 
region.getEncodedName());
+}
 if (htd.getPriority() >= HConstants.ADMIN_QOS || 
region.getTable().isSystemTable()) {
   regionServer.service.submit(new OpenPriorityRegionHandler(
 regionServer, regionServer, region, htd, masterSystemTime));

hbase git commit: HBASE-16354 Clean up defunct github PRs

2016-08-04 Thread busbey 
Repository: hbase
Updated Branches:
  refs/heads/master 975f0dd95 -> c9f84e813


HBASE-16354 Clean up defunct github PRs

* closes #34 mistaken branch-1.3 merge
* closes #32 no response from contributor in a month
* closes #31 no response from contributor in a month
* closes #30 no response from contributor in 2.5 months
* closes #29 fixed in HBASE-15863
* closes #25 no response from contributor in 4.5 months and obviated by 
HBASE-16340
* closes #24 no response from contributor in 4.5 months
* closes #22 jira HBASE-14442 closed as stale
* closes #16 fixed in HBASE-15574
* closes #13 no response from contributor in 4.5 months
* closes #11 no response from contributor in 4.5 months and obviated by 
HBASE-15265
* closes #4 no response from contributor in 4.5 months

Signed-off-by: Dima Spivak 
Signed-off-by: Esteban Gutierrez 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c9f84e81
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c9f84e81
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c9f84e81

Branch: refs/heads/master
Commit: c9f84e8137c451c797101146c9049f6dd440c662
Parents: 975f0dd
Author: Sean Busbey 
Authored: Thu Aug 4 02:02:56 2016 -0500
Committer: Sean Busbey 
Committed: Thu Aug 4 16:25:57 2016 -0500

--

--

hbase git commit: HBASE-9899 for idempotent operation dups, return the result instead of throwing conflict exception (Guanghao Zhang)

2016-08-04 Thread stack 
Repository: hbase
Updated Branches:
  refs/heads/master 550b937bc -> 975f0dd95


HBASE-9899 for idempotent operation dups, return the result instead of throwing 
conflict exception (Guanghao Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/975f0dd9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/975f0dd9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/975f0dd9

Branch: refs/heads/master
Commit: 975f0dd958debcdd842a95f8e9f7458689414fbf
Parents: 550b937
Author: stack 
Authored: Thu Aug 4 12:40:19 2016 -0700
Committer: stack 
Committed: Thu Aug 4 12:40:19 2016 -0700

--
 .../hadoop/hbase/protobuf/ProtobufUtil.java | 57 
 .../hadoop/hbase/regionserver/HRegion.java  | 53 ++
 .../hbase/regionserver/RSRpcServices.java   | 55 +++
 .../hadoop/hbase/regionserver/Region.java   | 11 
 .../hbase/regionserver/ServerNonceManager.java  | 43 +++
 .../hbase/client/HConnectionTestingUtility.java | 44 +++
 .../hadoop/hbase/client/TestFromClientSide.java | 45 
 .../client/TestIncrementsFromClientSide.java| 48 +
 .../hadoop/hbase/client/TestMultiParallel.java  | 14 +++--
 .../TestScannerHeartbeatMessages.java   |  2 +-
 10 files changed, 332 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/975f0dd9/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index b3bf041..5ba0572 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -859,6 +859,63 @@ public final class ProtobufUtil {
   }
 
   /**
+   * Convert a protocol buffer Mutate to a Get.
+   * @param proto the protocol buffer Mutate to convert.
+   * @param cellScanner
+   * @return the converted client get.
+   * @throws IOException
+   */
+  public static Get toGet(final MutationProto proto, final CellScanner 
cellScanner)
+  throws IOException {
+MutationType type = proto.getMutateType();
+assert type == MutationType.INCREMENT || type == MutationType.APPEND : 
type.name();
+byte[] row = proto.hasRow() ? proto.getRow().toByteArray() : null;
+Get get = null;
+int cellCount = proto.hasAssociatedCellCount() ? 
proto.getAssociatedCellCount() : 0;
+if (cellCount > 0) {
+  // The proto has metadata only and the data is separate to be found in 
the cellScanner.
+  if (cellScanner == null) {
+throw new DoNotRetryIOException("Cell count of " + cellCount + " but 
no cellScanner: "
++ TextFormat.shortDebugString(proto));
+  }
+  for (int i = 0; i < cellCount; i++) {
+if (!cellScanner.advance()) {
+  throw new DoNotRetryIOException("Cell count of " + cellCount + " but 
at index " + i
+  + " no cell returned: " + TextFormat.shortDebugString(proto));
+}
+Cell cell = cellScanner.current();
+if (get == null) {
+  get = new Get(Bytes.copy(cell.getRowArray(), cell.getRowOffset(), 
cell.getRowLength()));
+}
+get.addColumn(
+  Bytes.copy(cell.getFamilyArray(), cell.getFamilyOffset(), 
cell.getFamilyLength()),
+  Bytes.copy(cell.getQualifierArray(), cell.getQualifierOffset(),
+cell.getQualifierLength()));
+  }
+} else {
+  get = new Get(row);
+  for (ColumnValue column : proto.getColumnValueList()) {
+byte[] family = column.getFamily().toByteArray();
+for (QualifierValue qv : column.getQualifierValueList()) {
+  byte[] qualifier = qv.getQualifier().toByteArray();
+  if (!qv.hasValue()) {
+throw new DoNotRetryIOException("Missing required field: qualifier 
value");
+  }
+  get.addColumn(family, qualifier);
+}
+  }
+}
+if (proto.hasTimeRange()) {
+  TimeRange timeRange = protoToTimeRange(proto.getTimeRange());
+  get.setTimeRange(timeRange.getMin(), timeRange.getMax());
+}
+for (NameBytesPair attribute : proto.getAttributeList()) {
+  get.setAttribute(attribute.getName(), 
attribute.getValue().toByteArray());
+}
+return get;
+  }
+
+  /**
* Convert a client Scan to a protocol buffer Scan
*
* @param scan the client Scan to convert

http://git-wip-us.apache.org/repos/asf/hbase/blob/975f0dd9/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

hbase git commit: HBASE-16352 Port HBASE-15645 (hbase.rpc.timeout is not used in operations of HTable) to 0.98

2016-08-04 Thread apurtell 
Repository: hbase
Updated Branches:
  refs/heads/0.98 ee344b27f -> 342569620


HBASE-16352 Port HBASE-15645 (hbase.rpc.timeout is not used in operations of 
HTable) to 0.98


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/34256962
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/34256962
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/34256962

Branch: refs/heads/0.98
Commit: 3425696209b7c17aff4508985e0e316a9da57c5a
Parents: ee344b2
Author: Andrew Purtell 
Authored: Wed Aug 3 18:17:24 2016 -0700
Committer: Andrew Purtell 
Committed: Thu Aug 4 10:23:56 2016 -0700

--
 .../apache/hadoop/hbase/client/HBaseAdmin.java  | 48 +-
 .../org/apache/hadoop/hbase/client/HTable.java  | 53 ++--
 .../hadoop/hbase/client/RpcRetryingCaller.java  | 23 ++---
 .../hbase/client/RpcRetryingCallerFactory.java  | 20 +++-
 .../org/apache/hadoop/hbase/HConstants.java |  4 +-
 .../src/main/resources/hbase-default.xml| 11 +++-
 .../org/apache/hadoop/hbase/client/TestHCM.java | 29 +++
 7 files changed, 159 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/34256962/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index e3fcc6e..745da57 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -67,7 +67,9 @@ import 
org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
 import org.apache.hadoop.hbase.ipc.MasterCoprocessorRpcChannel;
+import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
 import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
+import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
@@ -175,8 +177,11 @@ public class HBaseAdmin implements Abortable, Closeable {
   private boolean aborted;
   private boolean cleanupConnectionOnClose = false; // close the connection in 
close()
   private boolean closed = false;
+  private int operationTimeout;
+  private int rpcTimeout;
 
   private RpcRetryingCallerFactory rpcCallerFactory;
+  private RpcControllerFactory rpcControllerFactory;
 
   /**
* Constructor.
@@ -211,6 +216,11 @@ public class HBaseAdmin implements Abortable, Closeable {
 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
 this.retryLongerMultiplier = this.conf.getInt(
 "hbase.client.retries.longer.multiplier", 10);
+this.operationTimeout = 
this.conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
+HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
+this.rpcTimeout = this.conf.getInt(HConstants.HBASE_RPC_TIMEOUT_KEY,
+HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+this.rpcControllerFactory = RpcControllerFactory.instantiate(this.conf);
 this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf,
   connection.getStatisticsTracker());
   }
@@ -420,7 +430,35 @@ public class HBaseAdmin implements Abortable, Closeable {
*/
   public HTableDescriptor getTableDescriptor(final TableName tableName)
   throws TableNotFoundException, IOException {
-return this.connection.getHTableDescriptor(tableName);
+ return getTableDescriptor(tableName, getConnection(), rpcCallerFactory, 
rpcControllerFactory,
+   operationTimeout, rpcTimeout);
+  }
+
+  static HTableDescriptor getTableDescriptor(final TableName tableName, 
HConnection connection,
+  RpcRetryingCallerFactory rpcCallerFactory, final RpcControllerFactory 
rpcControllerFactory,
+ int operationTimeout, int rpcTimeout) throws TableNotFoundException, 
IOException {
+
+  if (tableName == null) return null;
+  HTableDescriptor htd = executeCallable(new 
MasterCallable(connection) {
+@Override
+public HTableDescriptor call() throws ServiceException {
+  PayloadCarryingRpcController controller = 
rpcControllerFactory.newController();
+  controller.setPriority(tableName);
+  GetTableDescriptorsResponse htds;
+  GetTableDescriptorsRequest req =
+  RequestConverter.buildGetTableDescriptorsRequest(tableName);
+  htds =

hbase git commit: HBASE-16314 Retry on table snapshot failure (Vladimir Rodionov)

2016-08-04 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/HBASE-7912 e8e514181 -> 541740d48


HBASE-16314 Retry on table snapshot failure (Vladimir Rodionov)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/541740d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/541740d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/541740d4

Branch: refs/heads/HBASE-7912
Commit: 541740d48fe2d953f8a659d15848350af8d40177
Parents: e8e5141
Author: tedyu 
Authored: Thu Aug 4 10:21:46 2016 -0700
Committer: tedyu 
Committed: Thu Aug 4 10:21:46 2016 -0700

--
 .../backup/master/FullTableBackupProcedure.java | 54 +++-
 1 file changed, 41 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/541740d4/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
index 0a12b62..a7cfd8a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
@@ -35,13 +35,13 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.backup.BackupCopyService;
 import org.apache.hadoop.hbase.backup.BackupInfo;
 import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory;
 import org.apache.hadoop.hbase.backup.BackupType;
 import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.BackupCopyService.Type;
 import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
 import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
 import org.apache.hadoop.hbase.backup.impl.BackupException;
@@ -71,7 +71,13 @@ public class FullTableBackupProcedure
 extends StateMachineProcedure
 implements TableProcedureInterface {
   private static final Log LOG = 
LogFactory.getLog(FullTableBackupProcedure.class);
-
+  
+  private static final String SNAPSHOT_BACKUP_MAX_ATTEMPTS_KEY = 
"hbase.backup.snapshot.attempts.max";
+  private static final int DEFAULT_SNAPSHOT_BACKUP_MAX_ATTEMPTS = 10;
+  
+  private static final String SNAPSHOT_BACKUP_ATTEMPTS_DELAY_KEY = 
"hbase.backup.snapshot.attempts.delay";
+  private static final int DEFAULT_SNAPSHOT_BACKUP_ATTEMPTS_DELAY = 1;
+  
   private final AtomicBoolean aborted = new AtomicBoolean(false);
   private Configuration conf;
   private String backupId;
@@ -560,17 +566,7 @@ public class FullTableBackupProcedure
   LOG.debug("Unable to delete " + snapshotName, e);
 }
 // Kick off snapshot for backup
-try {
-  
env.getMasterServices().getSnapshotManager().takeSnapshot(backupSnapshot);
-} catch (IOException e) {
-  LOG.debug("Unable to take snapshot: " + snapshotName, e);
-}
-long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(
-env.getMasterConfiguration(),
-backupSnapshot.getType(), 
SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
-BackupServerUtil.waitForSnapshot(backupSnapshot, waitTime,
-  env.getMasterServices().getSnapshotManager(), 
env.getMasterConfiguration());
-// set the snapshot name in BackupStatus of this table, only after 
snapshot success.
+snapshotTable(env, backupSnapshot);  
 backupContext.setSnapshotName(tableName, backupSnapshot.getName());
   }
   setNextState(FullTableBackupState.SNAPSHOT_COPY);
@@ -621,6 +617,38 @@ public class FullTableBackupProcedure
 return Flow.HAS_MORE_STATE;
   }
 
+  private void snapshotTable(final MasterProcedureEnv env, SnapshotDescription 
backupSnapshot)
+throws IOException
+  {
+
+int maxAttempts = 
env.getMasterConfiguration().getInt(SNAPSHOT_BACKUP_MAX_ATTEMPTS_KEY, 
+  DEFAULT_SNAPSHOT_BACKUP_MAX_ATTEMPTS);
+int delay = 
env.getMasterConfiguration().getInt(SNAPSHOT_BACKUP_ATTEMPTS_DELAY_KEY, 
+  DEFAULT_SNAPSHOT_BACKUP_ATTEMPTS_DELAY);
+int attempts = 0;
+
+while (attempts++ < maxAttempts) {
+  try {
+
env.getMasterServices().getSnapshotManager().takeSnapshot(backupSnapshot);
+long waitTime =

hbase git commit: HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)

2016-08-04 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 dfc9616ae -> 3bce24dee


HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3bce24de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3bce24de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3bce24de

Branch: refs/heads/branch-1.2
Commit: 3bce24dee6edc3af9d3190a506e9d6df4c115108
Parents: dfc9616
Author: chenheng 
Authored: Thu Aug 4 21:13:42 2016 +0800
Committer: chenheng 
Committed: Thu Aug 4 21:37:48 2016 +0800

--
 .../hadoop/hbase/io/hfile/LruBlockCache.java| 36 ++--
 .../hadoop/hbase/io/hfile/TestCacheOnWrite.java |  2 ++
 .../hbase/io/hfile/TestLruBlockCache.java   |  5 +++
 3 files changed, 40 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 2781833..52ecf00 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -112,6 +112,10 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
*/
   static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.acceptable.factor";
 
+  /**
+   * Hard capacity limit of cache, will reject any put if size > this * 
acceptable
+   */
+  static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.hard.capacity.limit.factor";
   static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.single.percentage";
   static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.multi.percentage";
   static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.memory.percentage";
@@ -138,6 +142,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   static final float DEFAULT_MULTI_FACTOR = 0.50f;
   static final float DEFAULT_MEMORY_FACTOR = 0.25f;
 
+  /** default hard capacity limit */
+  static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f;
+
   static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
 
   /** Statistics thread */
@@ -171,6 +178,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   /** Cache access count (sequential ID) */
   private final AtomicLong count;
 
+  /** hard capacity limit */
+  private float hardCapacityLimitFactor;
+
   /** Cache statistics */
   private final CacheStats stats;
 
@@ -228,6 +238,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 DEFAULT_SINGLE_FACTOR,
 DEFAULT_MULTI_FACTOR,
 DEFAULT_MEMORY_FACTOR,
+DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
 false,
 DEFAULT_MAX_BLOCK_SIZE
 );
@@ -243,6 +254,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, 
DEFAULT_SINGLE_FACTOR),
 conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
 conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, 
DEFAULT_MEMORY_FACTOR),
+conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
 conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, 
DEFAULT_IN_MEMORY_FORCE_MODE),
 conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
 );
@@ -269,7 +281,8 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
   int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
   float minFactor, float acceptableFactor, float singleFactor,
-  float multiFactor, float memoryFactor, boolean forceInMemory, long 
maxBlockSize) {
+  float multiFactor, float memoryFactor, float hardLimitFactor,
+  boolean forceInMemory, long maxBlockSize) {
 this.maxBlockSize = maxBlockSize;
 if(singleFactor + multiFactor + memoryFactor != 1 ||
 singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
@@ -297,6 +310,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 this.elements = new AtomicLong(0);
 this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
 this.size = new AtomicLong(this.overhead);
+this.hardCapacityLimitFactor = hardLimitFactor;

hbase git commit: HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)

2016-08-04 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 af6ebcd1a -> f49d2fc7e


HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f49d2fc7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f49d2fc7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f49d2fc7

Branch: refs/heads/branch-1.3
Commit: f49d2fc7e1b3e3cbfee78a1a565ba8a03296d410
Parents: af6ebcd
Author: chenheng 
Authored: Thu Aug 4 21:13:42 2016 +0800
Committer: chenheng 
Committed: Thu Aug 4 21:31:55 2016 +0800

--
 .../hadoop/hbase/io/hfile/LruBlockCache.java| 36 ++--
 .../hadoop/hbase/io/hfile/TestCacheOnWrite.java |  2 ++
 .../hbase/io/hfile/TestLruBlockCache.java   |  5 +++
 3 files changed, 40 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f49d2fc7/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 268322a..09d82d0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -112,6 +112,10 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
*/
   static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.acceptable.factor";
 
+  /**
+   * Hard capacity limit of cache, will reject any put if size > this * 
acceptable
+   */
+  static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.hard.capacity.limit.factor";
   static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.single.percentage";
   static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.multi.percentage";
   static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.memory.percentage";
@@ -138,6 +142,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   static final float DEFAULT_MULTI_FACTOR = 0.50f;
   static final float DEFAULT_MEMORY_FACTOR = 0.25f;
 
+  /** default hard capacity limit */
+  static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f;
+
   static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
 
   /** Statistics thread */
@@ -171,6 +178,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   /** Cache access count (sequential ID) */
   private final AtomicLong count;
 
+  /** hard capacity limit */
+  private float hardCapacityLimitFactor;
+
   /** Cache statistics */
   private final CacheStats stats;
 
@@ -228,6 +238,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 DEFAULT_SINGLE_FACTOR,
 DEFAULT_MULTI_FACTOR,
 DEFAULT_MEMORY_FACTOR,
+DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
 false,
 DEFAULT_MAX_BLOCK_SIZE
 );
@@ -243,6 +254,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, 
DEFAULT_SINGLE_FACTOR),
 conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
 conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, 
DEFAULT_MEMORY_FACTOR),
+conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
 conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, 
DEFAULT_IN_MEMORY_FORCE_MODE),
 conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
 );
@@ -269,7 +281,8 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
   int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
   float minFactor, float acceptableFactor, float singleFactor,
-  float multiFactor, float memoryFactor, boolean forceInMemory, long 
maxBlockSize) {
+  float multiFactor, float memoryFactor, float hardLimitFactor,
+  boolean forceInMemory, long maxBlockSize) {
 this.maxBlockSize = maxBlockSize;
 if(singleFactor + multiFactor + memoryFactor != 1 ||
 singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
@@ -297,6 +310,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 this.elements = new AtomicLong(0);
 this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
 this.size = new AtomicLong(this.overhead);
+this.hardCapacityLimitFactor = hardLimitFactor;

hbase git commit: HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)

2016-08-04 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 737e06324 -> 9c8cbd44e


HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9c8cbd44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9c8cbd44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9c8cbd44

Branch: refs/heads/branch-1
Commit: 9c8cbd44edbf304e71ab87cd93ca90f136df5dd3
Parents: 737e063
Author: chenheng 
Authored: Thu Aug 4 21:13:42 2016 +0800
Committer: chenheng 
Committed: Thu Aug 4 21:23:27 2016 +0800

--
 .../hadoop/hbase/io/hfile/LruBlockCache.java| 36 ++--
 .../hadoop/hbase/io/hfile/TestCacheOnWrite.java |  2 ++
 .../hbase/io/hfile/TestLruBlockCache.java   |  5 +++
 3 files changed, 40 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9c8cbd44/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index cc91cc3..f427e04 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -112,6 +112,10 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
*/
   static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.acceptable.factor";
 
+  /**
+   * Hard capacity limit of cache, will reject any put if size > this * 
acceptable
+   */
+  static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.hard.capacity.limit.factor";
   static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.single.percentage";
   static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.multi.percentage";
   static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.memory.percentage";
@@ -138,6 +142,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   static final float DEFAULT_MULTI_FACTOR = 0.50f;
   static final float DEFAULT_MEMORY_FACTOR = 0.25f;
 
+  /** default hard capacity limit */
+  static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f;
+
   static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
 
   /** Statistics thread */
@@ -171,6 +178,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   /** Cache access count (sequential ID) */
   private final AtomicLong count;
 
+  /** hard capacity limit */
+  private float hardCapacityLimitFactor;
+
   /** Cache statistics */
   private final CacheStats stats;
 
@@ -228,6 +238,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 DEFAULT_SINGLE_FACTOR,
 DEFAULT_MULTI_FACTOR,
 DEFAULT_MEMORY_FACTOR,
+DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
 false,
 DEFAULT_MAX_BLOCK_SIZE
 );
@@ -243,6 +254,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, 
DEFAULT_SINGLE_FACTOR),
 conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
 conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, 
DEFAULT_MEMORY_FACTOR),
+conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
 conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, 
DEFAULT_IN_MEMORY_FORCE_MODE),
 conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
 );
@@ -269,7 +281,8 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
   int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
   float minFactor, float acceptableFactor, float singleFactor,
-  float multiFactor, float memoryFactor, boolean forceInMemory, long 
maxBlockSize) {
+  float multiFactor, float memoryFactor, float hardLimitFactor,
+  boolean forceInMemory, long maxBlockSize) {
 this.maxBlockSize = maxBlockSize;
 if(singleFactor + multiFactor + memoryFactor != 1 ||
 singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
@@ -297,6 +310,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 this.elements = new AtomicLong(0);
 this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
 this.size = new AtomicLong(this.overhead);
+this.hardCapacityLimitFactor = hardLimitFactor;

hbase git commit: HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)

2016-08-04 Thread chenheng 
Repository: hbase
Updated Branches:
  refs/heads/master 544dc1e55 -> 550b937bc


HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/550b937b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/550b937b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/550b937b

Branch: refs/heads/master
Commit: 550b937bcf0b5a7e030194cca7d90524a0fc0f3d
Parents: 544dc1e
Author: chenheng 
Authored: Thu Aug 4 21:13:42 2016 +0800
Committer: chenheng 
Committed: Thu Aug 4 21:13:42 2016 +0800

--
 .../hadoop/hbase/io/hfile/LruBlockCache.java| 36 ++--
 .../hadoop/hbase/io/hfile/TestCacheOnWrite.java |  2 ++
 .../hbase/io/hfile/TestLruBlockCache.java   |  5 +++
 3 files changed, 40 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/550b937b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 29c5922..2fd9fdf 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -110,6 +110,10 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
*/
   static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.acceptable.factor";
 
+  /**
+   * Hard capacity limit of cache, will reject any put if size > this * 
acceptable
+   */
+  static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = 
"hbase.lru.blockcache.hard.capacity.limit.factor";
   static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.single.percentage";
   static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.multi.percentage";
   static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = 
"hbase.lru.blockcache.memory.percentage";
@@ -136,6 +140,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   static final float DEFAULT_MULTI_FACTOR = 0.50f;
   static final float DEFAULT_MEMORY_FACTOR = 0.25f;
 
+  /** default hard capacity limit */
+  static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f;
+
   static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
 
   /** Statistics thread */
@@ -169,6 +176,9 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   /** Cache access count (sequential ID) */
   private final AtomicLong count;
 
+  /** hard capacity limit */
+  private float hardCapacityLimitFactor;
+
   /** Cache statistics */
   private final CacheStats stats;
 
@@ -226,6 +236,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 DEFAULT_SINGLE_FACTOR,
 DEFAULT_MULTI_FACTOR,
 DEFAULT_MEMORY_FACTOR,
+DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
 false,
 DEFAULT_MAX_BLOCK_SIZE
 );
@@ -241,6 +252,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, 
DEFAULT_SINGLE_FACTOR),
 conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
 conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, 
DEFAULT_MEMORY_FACTOR),
+conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
 conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, 
DEFAULT_IN_MEMORY_FORCE_MODE),
 conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
 );
@@ -267,7 +279,8 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
   int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
   float minFactor, float acceptableFactor, float singleFactor,
-  float multiFactor, float memoryFactor, boolean forceInMemory, long 
maxBlockSize) {
+  float multiFactor, float memoryFactor, float hardLimitFactor,
+  boolean forceInMemory, long maxBlockSize) {
 this.maxBlockSize = maxBlockSize;
 if(singleFactor + multiFactor + memoryFactor != 1 ||
 singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
@@ -295,6 +308,7 @@ public class LruBlockCache implements ResizableBlockCache, 
HeapSize {
 this.elements = new AtomicLong(0);
 this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
 this.size = new AtomicLong(this.overhead);
+this.hardCapacityLimitFactor = hardLimitFactor;

hbase git commit: HBASE-16355 Invalid hbase-common test-jar dependency scope

2016-08-04 Thread busbey 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 302979a2e -> af6ebcd1a


HBASE-16355 Invalid hbase-common test-jar dependency scope

hbase-client has hbase-common test-jar as dependency in compile scope, while it 
should be test scope instead.
This patch fixes the bug.

closes #12

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/af6ebcd1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/af6ebcd1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/af6ebcd1

Branch: refs/heads/branch-1.3
Commit: af6ebcd1a2ff4ab6f4ea63ed22b554b3d71186f0
Parents: 302979a
Author: Stevo Slavić 
Authored: Sun Apr 5 12:43:48 2015 +0200
Committer: Sean Busbey 
Committed: Thu Aug 4 01:33:05 2016 -0500

--
 hbase-client/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/af6ebcd1/hbase-client/pom.xml
--
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index a3ba28c..d8e1437 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -130,6 +130,7 @@
   org.apache.hbase
   hbase-common
   test-jar
+  test
 
 
   org.apache.hbase

hbase git commit: HBASE-16355 Invalid hbase-common test-jar dependency scope

2016-08-04 Thread busbey 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 96cce2dcc -> 737e06324


HBASE-16355 Invalid hbase-common test-jar dependency scope

hbase-client has hbase-common test-jar as dependency in compile scope, while it 
should be test scope instead.
This patch fixes the bug.

closes #12

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/737e0632
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/737e0632
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/737e0632

Branch: refs/heads/branch-1
Commit: 737e063240fddc429e01c75a37b9f4a902053c2e
Parents: 96cce2d
Author: Stevo Slavić 
Authored: Sun Apr 5 12:43:48 2015 +0200
Committer: Sean Busbey 
Committed: Thu Aug 4 01:29:25 2016 -0500

--
 hbase-client/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/737e0632/hbase-client/pom.xml
--
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 018beaa..dae9c67 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -130,6 +130,7 @@
   org.apache.hbase
   hbase-common
   test-jar
+  test
 
 
   org.apache.hbase

hbase git commit: HBASE-16355 Invalid hbase-common test-jar dependency scope

2016-08-04 Thread busbey 
Repository: hbase
Updated Branches:
  refs/heads/master 6fdf3aa21 -> 544dc1e55


HBASE-16355 Invalid hbase-common test-jar dependency scope

hbase-client has hbase-common test-jar as dependency in compile scope, while it 
should be test scope instead.
This patch fixes the bug.

closes #12

Signed-off-by: Sean Busbey 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/544dc1e5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/544dc1e5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/544dc1e5

Branch: refs/heads/master
Commit: 544dc1e55851e42a24bd14eb9efb9b21c2447d6b
Parents: 6fdf3aa
Author: Stevo Slavić 
Authored: Sun Apr 5 12:43:48 2015 +0200
Committer: Sean Busbey 
Committed: Thu Aug 4 01:17:51 2016 -0500

--
 hbase-client/pom.xml | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/544dc1e5/hbase-client/pom.xml
--
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index e74e0d5..7fc4672 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -130,6 +130,7 @@
   org.apache.hbase
   hbase-common
   test-jar
+  test
 
 
   org.apache.hbase