hbase git commit: HBASE-16209 provide an ExponentialBackOffPolicy sleep between failed region open requests

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 2b1cbeb28 -> fe4438361


HBASE-16209 provide an ExponentialBackOffPolicy sleep between failed region 
open requests

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fe443836
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fe443836
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fe443836

Branch: refs/heads/master
Commit: fe44383619d0d11396382f2bfa493ad93c8b0e5f
Parents: 2b1cbeb
Author: Joseph Hwang 
Authored: Tue Jul 26 11:13:32 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 29 09:47:00 2016 -0700

--
 .../master/AssignmentManagerStatusTmpl.jamon| 20 ++-
 .../hadoop/hbase/master/AssignmentManager.java  | 56 ++--
 .../hadoop/hbase/master/RegionStates.java   |  8 ++-
 .../hbase/master/TestMasterStatusServlet.java   |  8 ++-
 4 files changed, 84 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fe443836/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
index e2ae09d..06b320f 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
@@ -25,6 +25,8 @@ org.apache.hadoop.hbase.HBaseConfiguration;
 org.apache.hadoop.hbase.HConstants;
 java.util.HashSet;
 java.util.SortedSet;
+java.util.Map;
+java.util.concurrent.atomic.AtomicInteger;
 
 <%args>
 AssignmentManager assignmentManager;
@@ -32,7 +34,9 @@ int limit = 100;
 
 
 <%java SortedSet rit = assignmentManager
-  .getRegionStates().getRegionsInTransitionOrderedByTimestamp(); %>
+  .getRegionStates().getRegionsInTransitionOrderedByTimestamp();
+  Map failedRegionTracker = 
assignmentManager.getFailedOpenTracker();
+   %>
 
 <%if !rit.isEmpty() %>
 <%java>
@@ -83,7 +87,7 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / 
ritsPerPage);
  
  
  Region
- StateRIT time (ms)
+ StateRIT time (ms) Retries 

  
 
  <%if ritsOverThreshold.contains(rs.getRegion().getEncodedName()) 
%>
@@ -93,9 +97,21 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / 
ritsPerPage);
 <%else>
 
 
+<%java>
+  String retryStatus = "0";
+  String name = rs.getRegion().getEncodedName();
+  RegionState state = assignmentManager.getState(name);
+  AtomicInteger numOpenRetries = 
failedRegionTracker.get(name);
+  if (numOpenRetries != null ) {
+retryStatus = 
Integer.toString(numOpenRetries.get());
+  } else if (state.getState() ==  
RegionState.State.FAILED_OPEN) {
+ retryStatus = "Failed";
+  }
+
 <% rs.getRegion().getEncodedName() %>
 <% 
HRegionInfo.getDescriptiveNameFromRegionStateForDisplay(rs, conf) %>
 <% (currentTime - rs.getStamp()) %> 
+ <% retryStatus %> 
  
  <%java recordItr++; %>
  <%if (recordItr % ritsPerPage) == 0 %>

http://git-wip-us.apache.org/repos/asf/hbase/blob/fe443836/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 2ffe466..ffdbac8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -36,6 +36,7 @@ import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ 

hbase git commit: Back port HBASE-16209 Provide an exponential back off policy in reattempting failed region opens

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 08b9e6bee -> 7c97acf6e


Back port HBASE-16209 Provide an exponential back off policy in reattempting 
failed region opens

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c97acf6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c97acf6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c97acf6

Branch: refs/heads/branch-1
Commit: 7c97acf6e345023f043964d023816d5b3329dde9
Parents: 08b9e6b
Author: Joseph Hwang 
Authored: Wed Jul 13 09:38:37 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 29 09:47:30 2016 -0700

--
 .../master/AssignmentManagerStatusTmpl.jamon| 20 ++-
 .../hadoop/hbase/master/AssignmentManager.java  | 60 +++-
 .../hadoop/hbase/master/RegionStates.java   |  2 +-
 .../master/handler/ClosedRegionHandler.java |  2 +-
 .../hbase/master/TestMasterStatusServlet.java   |  2 +
 5 files changed, 80 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7c97acf6/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
index 67816b2..20a9ae3 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
@@ -30,6 +30,8 @@ java.util.ArrayList;
 java.util.Set;
 java.util.HashSet;
 java.lang.Integer;
+java.util.Map;
+java.util.concurrent.atomic.AtomicInteger;
 
 <%args>
 AssignmentManager assignmentManager;
@@ -37,7 +39,9 @@ int limit = Integer.MAX_VALUE;
 
 
 <%java SortedSet rit = assignmentManager
-  .getRegionStates().getRegionsInTransitionOrderedByTimestamp(); %>
+  .getRegionStates().getRegionsInTransitionOrderedByTimestamp();
+  Map failedRegionTracker = 
assignmentManager.getFailedOpenTracker();
+   %>
 
 <%if !rit.isEmpty() %>
 <%java>
@@ -92,7 +96,7 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / 
ritsPerPage);
   
   
   Region
-  StateRIT time (ms)
+  StateRIT time (ms)Retries 

   
 
   <%if ritsOverThreshold.contains(rs.getRegion().getEncodedName()) 
%>
@@ -102,9 +106,21 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / 
ritsPerPage);
   <%else>
   
   
+   <%java>
+  String retryStatus = "0";
+  String name = rs.getRegion().getEncodedName();
+  RegionState state = assignmentManager.getState(name);
+  AtomicInteger numOpenRetries = failedRegionTracker.get(name);
+  if (numOpenRetries != null ) {
+retryStatus = Integer.toString(numOpenRetries.get());
+  } else if (state.getState() ==  
RegionState.State.FAILED_OPEN) {
+ retryStatus = "Failed";
+  }
+
   <% rs.getRegion().getEncodedName() %>
   <% rs.toDescriptiveString() %>
   <% (currentTime - rs.getStamp()) %> 
+  <% retryStatus %> 
   <%java recordItr++; %>
   <%if (recordItr % ritsPerPage) == 0 %>
  

http://git-wip-us.apache.org/repos/asf/hbase/blob/7c97acf6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 2efc77d..b2e6e71 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -37,6 +37,7 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -100,6 +101,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.KeyLocker;
 import 

hbase git commit: HBASE-16096 Backport. Cleanly remove replication peers from ZooKeeper.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 7214bad10 -> 909f06147


HBASE-16096 Backport. Cleanly remove replication peers from ZooKeeper.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/909f0614
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/909f0614
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/909f0614

Branch: refs/heads/branch-1.3
Commit: 909f06147741f232d76bc411ffdd09f2627ca014
Parents: 7214bad
Author: Joseph Hwang 
Authored: Thu Jun 30 15:18:33 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jul 28 11:30:51 2016 -0700

--
 .../regionserver/ReplicationSourceManager.java  | 13 +--
 .../TestReplicationSourceManager.java   | 40 
 2 files changed, 50 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/909f0614/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index dcc3634..996c518 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -572,9 +572,10 @@ public class ReplicationSourceManager implements 
ReplicationListener {
   srcToRemove.add(src);
 }
   }
-  if (srcToRemove.size() == 0) {
-LOG.error("The queue we wanted to close is missing " + id);
-return;
+  if (srcToRemove.isEmpty()) {
+LOG.error("The peer we wanted to remove is missing a 
ReplicationSourceInterface. " +
+"This could mean that ReplicationSourceInterface initialization 
failed for this peer " +
+"and that replication on this peer may not be caught up. peerId=" 
+ id);
   }
   for (ReplicationSourceInterface toRemove : srcToRemove) {
 toRemove.terminate(terminateMessage);
@@ -752,6 +753,12 @@ public class ReplicationSourceManager implements 
ReplicationListener {
   }
 
   /**
+   * Get the ReplicationPeers used by this ReplicationSourceManager
+   * @return the ReplicationPeers used by this ReplicationSourceManager
+   */
+  public ReplicationPeers getReplicationPeers() {return this.replicationPeers;}
+
+  /**
* Get a string representation of all the sources' metrics
*/
   public String getStats() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/909f0614/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index f11e1e9..268d173 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -64,6 +64,7 @@ import 
org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.replication.ReplicationFactory;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import org.apache.hadoop.hbase.replication.ReplicationPeers;
 import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
 import org.apache.hadoop.hbase.replication.ReplicationQueues;
@@ -494,6 +495,45 @@ public class TestReplicationSourceManager {
   scopes.containsKey(f2));
   }
 
+  /**
+   * Test whether calling removePeer() on a ReplicationSourceManager that 
failed on initializing the
+   * corresponding ReplicationSourceInterface correctly cleans up the 
corresponding
+   * replication queue and ReplicationPeer.
+   * See HBASE-16096.
+   * @throws Exception
+   */
+  @Test
+  public void testPeerRemovalCleanup() throws Exception{
+String replicationSourceImplName = 
conf.get("replication.replicationsource.implementation");
+try {
+  DummyServer server = new DummyServer();
+  ReplicationQueues rq =
+  ReplicationFactory.getReplicationQueues(server.getZooKeeper(), 
server.getConfiguration(),
+  

hbase git commit: HBASE-16096 Backport. Cleanly remove replication peers from ZooKeeper.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 7983d2f45 -> d597e29ec


HBASE-16096 Backport. Cleanly remove replication peers from ZooKeeper.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d597e29e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d597e29e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d597e29e

Branch: refs/heads/branch-1
Commit: d597e29ec53fc87f1331f5f2fae3f564dfd963d3
Parents: 7983d2f
Author: Joseph Hwang 
Authored: Thu Jun 30 15:18:33 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jul 28 11:30:20 2016 -0700

--
 .../regionserver/ReplicationSourceManager.java  | 13 +--
 .../TestReplicationSourceManager.java   | 40 
 2 files changed, 50 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d597e29e/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 8ef581c..ab19223 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -573,9 +573,10 @@ public class ReplicationSourceManager implements 
ReplicationListener {
   srcToRemove.add(src);
 }
   }
-  if (srcToRemove.size() == 0) {
-LOG.error("The queue we wanted to close is missing " + id);
-return;
+  if (srcToRemove.isEmpty()) {
+LOG.error("The peer we wanted to remove is missing a 
ReplicationSourceInterface. " +
+"This could mean that ReplicationSourceInterface initialization 
failed for this peer " +
+"and that replication on this peer may not be caught up. peerId=" 
+ id);
   }
   for (ReplicationSourceInterface toRemove : srcToRemove) {
 toRemove.terminate(terminateMessage);
@@ -753,6 +754,12 @@ public class ReplicationSourceManager implements 
ReplicationListener {
   }
 
   /**
+   * Get the ReplicationPeers used by this ReplicationSourceManager
+   * @return the ReplicationPeers used by this ReplicationSourceManager
+   */
+  public ReplicationPeers getReplicationPeers() {return this.replicationPeers;}
+
+  /**
* Get a string representation of all the sources' metrics
*/
   public String getStats() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/d597e29e/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index f11e1e9..268d173 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -64,6 +64,7 @@ import 
org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.replication.ReplicationFactory;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import org.apache.hadoop.hbase.replication.ReplicationPeers;
 import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
 import org.apache.hadoop.hbase.replication.ReplicationQueues;
@@ -494,6 +495,45 @@ public class TestReplicationSourceManager {
   scopes.containsKey(f2));
   }
 
+  /**
+   * Test whether calling removePeer() on a ReplicationSourceManager that 
failed on initializing the
+   * corresponding ReplicationSourceInterface correctly cleans up the 
corresponding
+   * replication queue and ReplicationPeer.
+   * See HBASE-16096.
+   * @throws Exception
+   */
+  @Test
+  public void testPeerRemovalCleanup() throws Exception{
+String replicationSourceImplName = 
conf.get("replication.replicationsource.implementation");
+try {
+  DummyServer server = new DummyServer();
+  ReplicationQueues rq =
+  ReplicationFactory.getReplicationQueues(server.getZooKeeper(), 
server.getConfiguration(),
+  

hbase git commit: HBASE-16262 Backport HBASE-13839: Fix AssgnmentManagerTmpl.jamon issues (coloring, content …etc). Ignoring UI changes provided by HBASE-11344.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 e06bbe480 -> d349f642c


HBASE-16262 Backport HBASE-13839: Fix AssgnmentManagerTmpl.jamon issues 
(coloring, content …etc). Ignoring UI changes provided by HBASE-11344.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d349f642
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d349f642
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d349f642

Branch: refs/heads/branch-1
Commit: d349f642c02c83eaa020afcee076f16cba96e299
Parents: e06bbe4
Author: Joseph Hwang 
Authored: Fri Feb 12 15:06:38 2016 -0800
Committer: Elliott Clark 
Committed: Wed Jul 27 13:09:28 2016 -0700

--
 .../master/AssignmentManagerStatusTmpl.jamon| 146 ---
 .../hadoop/hbase/master/RegionStates.java   |  29 +++-
 .../hbase/master/TestMasterStatusServlet.java   |   6 +-
 3 files changed, 122 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d349f642/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
index 64bb7ef..67816b2 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
@@ -23,81 +23,115 @@ org.apache.hadoop.hbase.master.RegionState;
 org.apache.hadoop.conf.Configuration;
 org.apache.hadoop.hbase.HBaseConfiguration;
 org.apache.hadoop.hbase.HConstants;
+java.util.SortedSet;
 java.util.Iterator;
+java.util.List;
+java.util.ArrayList;
 java.util.Set;
+java.util.HashSet;
+java.lang.Integer;
 
 <%args>
 AssignmentManager assignmentManager;
-int limit = 100;
+int limit = Integer.MAX_VALUE;
 
+
+<%java SortedSet rit = assignmentManager
+  .getRegionStates().getRegionsInTransitionOrderedByTimestamp(); %>
+
+<%if !rit.isEmpty() %>
 <%java>
-Set rit = 
assignmentManager.getRegionStates().getRegionsInTransition();
+Set ritsOverThreshold = new HashSet<>();
+Set ritsTwiceThreshold = new HashSet<>();
 // process the map to find region in transition details
 Configuration conf = HBaseConfiguration.create();
 int ritThreshold = conf.getInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 
6);
 int numOfRITOverThreshold = 0;
-long maxRITTime = Long.MIN_VALUE;
 long currentTime = System.currentTimeMillis();
-String regionIDForOldestRIT = ""; // avoiding null
+List ritToDisplay = new ArrayList<>();
+int count = 0;
 for (RegionState rs : rit) {
   long ritTime = currentTime - rs.getStamp();
-  if(ritTime > ritThreshold) {
+  if(ritTime > (ritThreshold * 2)) {
  numOfRITOverThreshold++;
-   }
-   if(maxRITTime < ritTime) {
- maxRITTime = ritTime;
- regionIDForOldestRIT = rs.getRegion().getEncodedName();
-   }
-}
-
-int totalRITs = rit.size();
-int toRemove = rit.size() - limit;
-int removed = 0;
-if (toRemove > 0) {
-  // getRegionsInTransition returned a copy, so we can mutate it
-  for (Iterator it = rit.iterator(); it.hasNext() && toRemove > 
0;) {
-RegionState rs = it.next();
-String regionEncodedName = rs.getRegion().getEncodedName();
-if 
(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals(regionEncodedName) ||
- regionIDForOldestRIT.equals(regionEncodedName)) {
-  // don't remove the meta & the oldest rit regions, they're too 
interesting!
-  continue;
-}
-it.remove();
-toRemove--;
-removed++;
+ ritsTwiceThreshold.add(rs.getRegion().getEncodedName());
+  } else if (ritTime > ritThreshold) {
+ numOfRITOverThreshold++;
+ ritsOverThreshold.add(rs.getRegion().getEncodedName());
+  }
+  if (count++ < limit) {
+ritToDisplay.add(rs);
   }
 }
-
+int numOfRITs = rit.size();
+int ritsPerPage = Math.min(5, numOfRITs);
+int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage);
 
-
-
-<%if !rit.isEmpty() %>
 
 Regions in Transition
-
-RegionStateRIT time (ms)
-<%for RegionState rs : rit %>
-<%if regionIDForOldestRIT.equals(rs.getRegion().getEncodedName()) 
%>
-
-<%else>
-
-
-<% rs.getRegion().getEncodedName() %><% 
rs.toDescriptiveString() %>
-<% (currentTime - rs.getStamp()) %> 
+ <% numOfRITs %> region(s) in transition.
+ <%if !ritsTwiceThreshold.isEmpty()  %>
+ 
+ <%elseif !ritsOverThreshold.isEmpty() %>
+ 
+ 

hbase git commit: HBASE-16096 Properly remove the replication queue and peer znodes after calling ReplicationSourceManager.removePeer().

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 03fe257a6 -> 744248c13


HBASE-16096 Properly remove the replication queue and peer znodes after calling 
ReplicationSourceManager.removePeer().

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/744248c1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/744248c1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/744248c1

Branch: refs/heads/master
Commit: 744248c131d344e5ddab3cfe032aad919dc0de0f
Parents: 03fe257
Author: Joseph Hwang 
Authored: Thu Jun 30 15:18:33 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 22 18:11:41 2016 -0700

--
 .../regionserver/ReplicationSourceManager.java  | 15 +---
 .../TestReplicationSourceManager.java   | 40 
 2 files changed, 50 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/744248c1/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index e2a232f..586aace 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -561,9 +561,10 @@ public class ReplicationSourceManager implements 
ReplicationListener {
   srcToRemove.add(src);
 }
   }
-  if (srcToRemove.size() == 0) {
-LOG.error("The queue we wanted to close is missing " + id);
-return;
+  if (srcToRemove.isEmpty()) {
+LOG.error("The peer we wanted to remove is missing a 
ReplicationSourceInterface. " +
+"This could mean that ReplicationSourceInterface initialization 
failed for this peer " +
+"and that replication on this peer may not be caught up. peerId=" 
+ id);
   }
   for (ReplicationSourceInterface toRemove : srcToRemove) {
 toRemove.terminate(terminateMessage);
@@ -739,8 +740,6 @@ public class ReplicationSourceManager implements 
ReplicationListener {
 }
   }
 
-
-
   /**
* Get the directory where wals are archived
* @return the directory where wals are archived
@@ -766,6 +765,12 @@ public class ReplicationSourceManager implements 
ReplicationListener {
   }
 
   /**
+   * Get the ReplicationPeers used by this ReplicationSourceManager
+   * @return the ReplicationPeers used by this ReplicationSourceManager
+   */
+  public ReplicationPeers getReplicationPeers() {return this.replicationPeers;}
+
+  /**
* Get a string representation of all the sources' metrics
*/
   public String getStats() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/744248c1/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 4442bbb..7696e95 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -64,6 +64,7 @@ import 
org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.replication.ReplicationFactory;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
 import org.apache.hadoop.hbase.replication.ReplicationPeers;
 import org.apache.hadoop.hbase.replication.ReplicationQueues;
 import org.apache.hadoop.hbase.replication.ReplicationQueuesArguments;
@@ -424,6 +425,45 @@ public abstract class TestReplicationSourceManager {
   scopes.containsKey(f2));
   }
 
+  /**
+   * Test whether calling removePeer() on a ReplicationSourceManager that 
failed on initializing the
+   * corresponding ReplicationSourceInterface correctly cleans up the 
corresponding
+   * replication queue and ReplicationPeer.
+   * See HBASE-16096.
+   * @throws Exception
+   */
+  @Test
+  public void testPeerRemovalCleanup() throws Exception{
+String 

hbase git commit: HBASE-16221 Have ThriftScanner update the ConnectionCache's last used time overtime getScannerRow() to keep the connection alive for long lived scanners

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 24a3d6952 -> 276acdb0b


HBASE-16221 Have ThriftScanner update the ConnectionCache's last used time 
overtime getScannerRow() to keep the connection alive for long lived scanners

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/276acdb0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/276acdb0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/276acdb0

Branch: refs/heads/branch-1
Commit: 276acdb0b0f62a4909c92334755c15c48da19595
Parents: 24a3d69
Author: Joseph Hwang 
Authored: Fri Jul 22 10:38:43 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 22 17:58:56 2016 -0700

--
 .../hadoop/hbase/util/ConnectionCache.java  | 14 ++
 .../thrift2/ThriftHBaseServiceHandler.java  |  2 +-
 .../thrift2/TestThriftHBaseServiceHandler.java  | 51 
 3 files changed, 66 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/276acdb0/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
index 1475879..ec5294b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
@@ -189,6 +189,20 @@ public class ConnectionCache {
 return connInfo;
   }
 
+  /**
+   * Updates the access time for the current connection. Used to keep 
Connections alive for
+   * long-lived scanners.
+   * @return whether we successfully updated the last access time
+   */
+  public boolean updateConnectionAccessTime() {
+String userName = getEffectiveUser();
+ConnectionInfo connInfo = connections.get(userName);
+if (connInfo != null) {
+  return connInfo.updateAccessTime();
+}
+return false;
+  }
+
   class ConnectionInfo {
 final Connection connection;
 final String userName;

http://git-wip-us.apache.org/repos/asf/hbase/blob/276acdb0/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
--
diff --git 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index c59576e..5900952 100644
--- 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -382,8 +382,8 @@ public class ThriftHBaseServiceHandler implements 
THBaseService.Iface {
   ex.setMessage("Invalid scanner Id");
   throw ex;
 }
-
 try {
+  connectionCache.updateConnectionAccessTime();
   return resultsFromHBase(scanner.next(numRows));
 } catch (IOException e) {
   throw getTIOError(e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/276acdb0/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
--
diff --git 
a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
 
b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 568ab0f..2a59b87 100644
--- 
a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ 
b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -589,6 +589,57 @@ public class TestThriftHBaseServiceHandler {
 }
   }
 
+  /**
+   * Tests keeping a HBase scanner alive for long periods of time. Each call 
to getScannerRow()
+   * should reset the ConnectionCache timeout for the scanner's connection
+   * @throws Exception
+   */
+  @Test
+  public void testLongLivedScan() throws Exception {
+int numTrials = 6;
+int trialPause = 1000;
+int cleanUpInterval = 100;
+Configuration conf = new Configuration(UTIL.getConfiguration());
+// Set the ConnectionCache timeout to trigger halfway through the trials
+conf.setInt(ThriftHBaseServiceHandler.MAX_IDLETIME, (numTrials / 2) * 
trialPause);
+conf.setInt(ThriftHBaseServiceHandler.CLEANUP_INTERVAL, cleanUpInterval);
+ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(conf,
+UserProvider.instantiate(conf));
+
+ByteBuffer table = wrap(tableAname);
+// insert data
+TColumnValue 

hbase git commit: HBASE-16221 Have ThriftScanner update the ConnectionCache's last used time overtime getScannerRow() to keep the connection alive for long lived scanners

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 ad91bfd0a -> 378541722


HBASE-16221 Have ThriftScanner update the ConnectionCache's last used time 
overtime getScannerRow() to keep the connection alive for long lived scanners

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/37854172
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/37854172
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/37854172

Branch: refs/heads/branch-1.3
Commit: 3785417224f7211d75a345e33a08af9e90802ee1
Parents: ad91bfd
Author: Joseph Hwang 
Authored: Fri Jul 22 10:38:43 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 22 17:59:21 2016 -0700

--
 .../hadoop/hbase/util/ConnectionCache.java  | 14 ++
 .../thrift2/ThriftHBaseServiceHandler.java  |  2 +-
 .../thrift2/TestThriftHBaseServiceHandler.java  | 51 
 3 files changed, 66 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/37854172/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
index 1475879..ec5294b 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
@@ -189,6 +189,20 @@ public class ConnectionCache {
 return connInfo;
   }
 
+  /**
+   * Updates the access time for the current connection. Used to keep 
Connections alive for
+   * long-lived scanners.
+   * @return whether we successfully updated the last access time
+   */
+  public boolean updateConnectionAccessTime() {
+String userName = getEffectiveUser();
+ConnectionInfo connInfo = connections.get(userName);
+if (connInfo != null) {
+  return connInfo.updateAccessTime();
+}
+return false;
+  }
+
   class ConnectionInfo {
 final Connection connection;
 final String userName;

http://git-wip-us.apache.org/repos/asf/hbase/blob/37854172/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
--
diff --git 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index c59576e..5900952 100644
--- 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -382,8 +382,8 @@ public class ThriftHBaseServiceHandler implements 
THBaseService.Iface {
   ex.setMessage("Invalid scanner Id");
   throw ex;
 }
-
 try {
+  connectionCache.updateConnectionAccessTime();
   return resultsFromHBase(scanner.next(numRows));
 } catch (IOException e) {
   throw getTIOError(e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/37854172/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
--
diff --git 
a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
 
b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 568ab0f..2a59b87 100644
--- 
a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ 
b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -589,6 +589,57 @@ public class TestThriftHBaseServiceHandler {
 }
   }
 
+  /**
+   * Tests keeping a HBase scanner alive for long periods of time. Each call 
to getScannerRow()
+   * should reset the ConnectionCache timeout for the scanner's connection
+   * @throws Exception
+   */
+  @Test
+  public void testLongLivedScan() throws Exception {
+int numTrials = 6;
+int trialPause = 1000;
+int cleanUpInterval = 100;
+Configuration conf = new Configuration(UTIL.getConfiguration());
+// Set the ConnectionCache timeout to trigger halfway through the trials
+conf.setInt(ThriftHBaseServiceHandler.MAX_IDLETIME, (numTrials / 2) * 
trialPause);
+conf.setInt(ThriftHBaseServiceHandler.CLEANUP_INTERVAL, cleanUpInterval);
+ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(conf,
+UserProvider.instantiate(conf));
+
+ByteBuffer table = wrap(tableAname);
+// insert data
+

hbase git commit: HBASE-16221 Have ThriftScanner update the ConnectionCache's last used time overtime getScannerRow() to keep the connection alive for long lived scanners

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master cc766df28 -> 03fe257a6


HBASE-16221 Have ThriftScanner update the ConnectionCache's last used time 
overtime getScannerRow() to keep the connection alive for long lived scanners

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/03fe257a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/03fe257a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/03fe257a

Branch: refs/heads/master
Commit: 03fe257a641bc42ed98fdff3914950cfe9045f66
Parents: cc766df
Author: Joseph Hwang 
Authored: Fri Jul 22 10:38:43 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 22 17:58:39 2016 -0700

--
 .../hadoop/hbase/util/ConnectionCache.java  | 14 ++
 .../thrift2/ThriftHBaseServiceHandler.java  |  2 +-
 .../thrift2/TestThriftHBaseServiceHandler.java  | 51 
 3 files changed, 66 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/03fe257a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
index a860f20..0659a0d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConnectionCache.java
@@ -189,6 +189,20 @@ public class ConnectionCache {
 return connInfo;
   }
 
+  /**
+   * Updates the access time for the current connection. Used to keep 
Connections alive for
+   * long-lived scanners.
+   * @return whether we successfully updated the last access time
+   */
+  public boolean updateConnectionAccessTime() {
+String userName = getEffectiveUser();
+ConnectionInfo connInfo = connections.get(userName);
+if (connInfo != null) {
+  return connInfo.updateAccessTime();
+}
+return false;
+  }
+
   class ConnectionInfo {
 final Connection connection;
 final String userName;

http://git-wip-us.apache.org/repos/asf/hbase/blob/03fe257a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
--
diff --git 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
index 9dea9a5..a69a7df 100644
--- 
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
+++ 
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftHBaseServiceHandler.java
@@ -383,8 +383,8 @@ public class ThriftHBaseServiceHandler implements 
THBaseService.Iface {
   ex.setMessage("Invalid scanner Id");
   throw ex;
 }
-
 try {
+  connectionCache.updateConnectionAccessTime();
   return resultsFromHBase(scanner.next(numRows));
 } catch (IOException e) {
   throw getTIOError(e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/03fe257a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
--
diff --git 
a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
 
b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
index 6fca282..38e3780 100644
--- 
a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
+++ 
b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java
@@ -589,6 +589,57 @@ public class TestThriftHBaseServiceHandler {
 }
   }
 
+  /**
+   * Tests keeping a HBase scanner alive for long periods of time. Each call 
to getScannerRow()
+   * should reset the ConnectionCache timeout for the scanner's connection
+   * @throws Exception
+   */
+  @Test
+  public void testLongLivedScan() throws Exception {
+int numTrials = 6;
+int trialPause = 1000;
+int cleanUpInterval = 100;
+Configuration conf = new Configuration(UTIL.getConfiguration());
+// Set the ConnectionCache timeout to trigger halfway through the trials
+conf.setInt(ThriftHBaseServiceHandler.MAX_IDLETIME, (numTrials / 2) * 
trialPause);
+conf.setInt(ThriftHBaseServiceHandler.CLEANUP_INTERVAL, cleanUpInterval);
+ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(conf,
+UserProvider.instantiate(conf));
+
+ByteBuffer table = wrap(tableAname);
+// insert data
+TColumnValue 

hbase git commit: HBASE-16208 Check that the row exists before attempting to remove a queue from TableBasedReplicationQueuesImpl

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 6d94925af -> f292048ff


HBASE-16208 Check that the row exists before attempting to remove a queue from 
TableBasedReplicationQueuesImpl

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f292048f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f292048f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f292048f

Branch: refs/heads/master
Commit: f292048ffd91503360cb56bb9b5f60fd47e2ebe0
Parents: 6d94925
Author: Joseph Hwang 
Authored: Mon Jul 11 10:45:53 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 12 14:50:07 2016 -0700

--
 .../replication/TableBasedReplicationQueuesImpl.java  | 10 +++---
 .../hbase/replication/TestReplicationStateHBaseImpl.java  |  7 ++-
 2 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f292048f/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
index 28fa967..3ee6fde 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
@@ -93,11 +93,15 @@ public class TableBasedReplicationQueuesImpl extends 
ReplicationTableBase
 
   @Override
   public void removeQueue(String queueId) {
-
 try {
   byte[] rowKey = queueIdToRowKey(queueId);
-  Delete deleteQueue = new Delete(rowKey);
-  safeQueueUpdate(deleteQueue);
+  if (checkQueueExists(queueId)) {
+Delete deleteQueue = new Delete(rowKey);
+safeQueueUpdate(deleteQueue);
+  } else {
+LOG.info("No logs were registered for queue id=" + queueId + " so no 
rows were removed " +
+"from the replication table while removing the queue");
+  }
 } catch (IOException | ReplicationException e) {
   String errMsg = "Failed removing queue queueId=" + queueId;
   abortable.abort(errMsg, e);

http://git-wip-us.apache.org/repos/asf/hbase/blob/f292048f/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
index ccae8a5..7ec6df8 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
@@ -84,7 +84,7 @@ public class TestReplicationStateHBaseImpl {
 conf.setClass("hbase.region.replica.replication.replicationQueues.class",
   TableBasedReplicationQueuesImpl.class, ReplicationQueues.class);
 
conf.setClass("hbase.region.replica.replication.replicationQueuesClient.class",
-  TableBasedReplicationQueuesClientImpl.class, 
ReplicationQueuesClient.class);
+TableBasedReplicationQueuesClientImpl.class, 
ReplicationQueuesClient.class);
 utility.startMiniCluster();
 zkw = HBaseTestingUtility.getZooKeeperWatcher(utility);
 String replicationZNodeName = conf.get("zookeeper.znode.replication", 
"replication");
@@ -195,6 +195,11 @@ public class TestReplicationStateHBaseImpl {
   assertNull(rq1.getLogsInQueue("Queue1"));
   // Test that getting logs from a non-existent queue aborts
   assertEquals(6, ds1.getAbortCount());
+  // Test removing a non-existent queue does not cause an abort. This is 
because we can
+  // attempt to remove a queue that has no corresponding Replication Table 
row (if we never
+  // registered a WAL for it)
+  rq1.removeQueue("NotHereQueue");
+  assertEquals(6, ds1.getAbortCount());
 } catch (ReplicationException e) {
   e.printStackTrace();
   fail("testAddLog received a ReplicationException");

hbase git commit: HBASE-16214 Add in a UI description for the Replication Table

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 8cfaa0e72 -> 90ba723dc


HBASE-16214 Add in a UI description for the Replication Table

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/90ba723d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/90ba723d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/90ba723d

Branch: refs/heads/master
Commit: 90ba723dc63571cef14546e9988a76f0adb923aa
Parents: 8cfaa0e
Author: Joseph Hwang 
Authored: Tue Jul 12 09:45:05 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 12 12:21:55 2016 -0700

--
 .../apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon| 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/90ba723d/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 056c2d7..d19df21 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -387,7 +387,7 @@ AssignmentManager assignmentManager = 
master.getAssignmentManager();
 description = "The hbase:canary table is used to sniff the write 
availbility of"
   + " each regionserver.";
 } else if (tableName.equals(AccessControlLists.ACL_TABLE_NAME)){
-description = "The hbase:acl table holds information about acl";
+description = "The hbase:acl table holds information about acl.";
 } else if (tableName.equals(VisibilityConstants.LABELS_TABLE_NAME)){
 description = "The hbase:labels table holds information about 
visibility labels.";
 } else if (tableName.equals(TableName.NAMESPACE_TABLE_NAME)){
@@ -396,7 +396,10 @@ AssignmentManager assignmentManager = 
master.getAssignmentManager();
 description = "The hbase:quota table holds quota information about 
number" +
 " or size of requests in a given time frame.";
 } else if (tableName.equals(TableName.valueOf("hbase:rsgroup"))){
-description = "The hbase:rsgroup table holds information about 
regionserver groups";
+description = "The hbase:rsgroup table holds information about 
regionserver groups.";
+} else if (tableName.equals(TableName.valueOf("hbase:replication"))) {
+description = "The hbase:replication table tracks cross cluster 
replication through " +
+"WAL file offsets.";
 }
 
 <% description %>

hbase git commit: HBASE-16216 Clean up source code.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 0eb150f9d -> 6cadc3828


HBASE-16216 Clean up source code.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6cadc382
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6cadc382
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6cadc382

Branch: refs/heads/HBASE-14850
Commit: 6cadc3828b714c31380df8f274c9a7fd6916a475
Parents: 0eb150f
Author: Elliott Clark 
Authored: Tue Jul 12 12:21:11 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 12 13:01:37 2016 -0700

--
 hbase-native-client/core/cell-test.cc |  8 +++---
 hbase-native-client/core/cell.cc  | 40 +++---
 hbase-native-client/core/cell.h   |  4 +--
 hbase-native-client/if/HBase.proto|  5 
 hbase-native-client/test-util/test-util.h |  2 +-
 hbase-native-client/third-party/BUCK  |  6 ++--
 hbase-native-client/utils/user-util.h |  2 +-
 7 files changed, 21 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6cadc382/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
index 49f567b..1d5c0eb 100644
--- a/hbase-native-client/core/cell-test.cc
+++ b/hbase-native-client/core/cell-test.cc
@@ -19,9 +19,9 @@
 
 #include "core/cell.h"
 
-#include 
-#include 
 #include 
+#include 
+#include 
 
 using namespace hbase;
 TEST(CellTest, CellFailureTest) {
@@ -74,8 +74,8 @@ TEST(CellTest, MultipleCellsTest) {
 row += std::to_string(i);
 value += std::to_string(i);
 CellType cell_type = CellType::PUT;
-const Cell *cell = new Cell(row, family, column, timestamp, value,
-cell_type);
+const Cell *cell =
+new Cell(row, family, column, timestamp, value, cell_type);
 cells.push_back(cell);
   }
   int i = 0;

http://git-wip-us.apache.org/repos/asf/hbase/blob/6cadc382/hbase-native-client/core/cell.cc
--
diff --git a/hbase-native-client/core/cell.cc b/hbase-native-client/core/cell.cc
index f214479..5129bc9 100644
--- a/hbase-native-client/core/cell.cc
+++ b/hbase-native-client/core/cell.cc
@@ -25,13 +25,8 @@ namespace hbase {
 Cell::Cell(const std::string , const std::string ,
const std::string , const long ,
const std::string , const hbase::CellType _type)
-: row_(row),
-  family_(family),
-  qualifier_(qualifier),
-  timestamp_(timestamp),
-  cell_type_(cell_type),
-  value_(value),
-  sequence_id_(0) {
+: row_(row), family_(family), qualifier_(qualifier), timestamp_(timestamp),
+  cell_type_(cell_type), value_(value), sequence_id_(0) {
 
   if (0 == row.size())
 throw std::runtime_error("Row size should be greater than 0");
@@ -43,35 +38,20 @@ Cell::Cell(const std::string , const std::string 
,
 throw std::runtime_error("Timestamp should be greater than 0");
 }
 
-Cell::~Cell() {
-}
+Cell::~Cell() {}
 
-const std::string ::Row() const {
-  return row_;
-}
+const std::string ::Row() const { return row_; }
 
-const std::string ::Family() const {
-  return family_;
-}
+const std::string ::Family() const { return family_; }
 
-const std::string ::Qualifier() const {
-  return qualifier_;
-}
+const std::string ::Qualifier() const { return qualifier_; }
 
-unsigned long Cell::Timestamp() const {
-  return timestamp_;
-}
+unsigned long Cell::Timestamp() const { return timestamp_; }
 
-const std::string ::Value() const {
-  return value_;
-}
+const std::string ::Value() const { return value_; }
 
-hbase::CellType Cell::Type() const {
-  return cell_type_;
-}
+hbase::CellType Cell::Type() const { return cell_type_; }
 
-long Cell::SequenceId() const {
-  return sequence_id_;
-}
+long Cell::SequenceId() const { return sequence_id_; }
 
 } /* namespace hbase */

http://git-wip-us.apache.org/repos/asf/hbase/blob/6cadc382/hbase-native-client/core/cell.h
--
diff --git a/hbase-native-client/core/cell.h b/hbase-native-client/core/cell.h
index 16ed280..2b15ad6 100644
--- a/hbase-native-client/core/cell.h
+++ b/hbase-native-client/core/cell.h
@@ -34,7 +34,7 @@ enum CellType {
 };
 
 class Cell {
- public:
+public:
   Cell(const std::string , const std::string ,
const std::string , const long ,
const std::string , const hbase::CellType _type);
@@ -47,7 +47,7 @@ class Cell {
   CellType Type() const;
   long SequenceId() const;
 
- private:
+private:
   std::string row_;
   std::string family_;
   std::string qualifier_;

[37/50] [abbrv] hbase git commit: HBASE-15748 Don't link in static libunwind.

[eclark]

HBASE-15748 Don't link in static libunwind.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50d16726
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50d16726
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50d16726

Branch: refs/heads/HBASE-14850
Commit: 50d16726ccf5b93a318417ab5b69d083c6f6ca8d
Parents: f74379e
Author: Elliott Clark 
Authored: Mon May 2 11:41:02 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/third-party/BUCK | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/50d16726/hbase-native-client/third-party/BUCK
--
diff --git a/hbase-native-client/third-party/BUCK 
b/hbase-native-client/third-party/BUCK
index 4327530..6561b2a 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -51,7 +51,7 @@ def add_dynamic_libs(names=[]):
 return rules
 
 
-system_libs = ["unwind", "lzma", "event", ]
+system_libs = ["lzma", "event", ]
 local_libs = [
 "double-conversion",
 "boost_regex",
@@ -65,7 +65,7 @@ local_libs = [
 "glog",
 "protobuf",
 ]
-dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ]
+dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", "unwind"]
 dynamic_rules = add_dynamic_libs(dynamic_libs)
 tp_dep_rules =  add_system_libs(system_libs,) \
   + add_system_libs(local_libs, lib_dir = "/usr/local/lib") \

[44/50] [abbrv] hbase git commit: HBASE-15730 Add on script to format all .h, .cc, and BUCK files.

[eclark]

HBASE-15730 Add on script to format all .h,.cc, and BUCK files.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1140d74
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1140d74
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1140d74

Branch: refs/heads/HBASE-14850
Commit: c1140d742b5fa0ec6270479f580d58b0a45c2a17
Parents: 14a33d7
Author: Elliott Clark 
Authored: Wed Apr 27 15:52:27 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/format-code.sh  |  7 +++
 hbase-native-client/connection/request.cc   | 14 +++---
 hbase-native-client/utils/BUCK  |  8 
 hbase-native-client/utils/user-util-test.cc |  2 +-
 hbase-native-client/utils/user-util.cc  |  4 ++--
 hbase-native-client/utils/user-util.h   |  2 +-
 6 files changed, 22 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/bin/format-code.sh
--
diff --git a/hbase-native-client/bin/format-code.sh 
b/hbase-native-client/bin/format-code.sh
new file mode 100755
index 000..24d6515
--- /dev/null
+++ b/hbase-native-client/bin/format-code.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+
+find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 
clang-format -i
+find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/connection/request.cc
--
diff --git a/hbase-native-client/connection/request.cc 
b/hbase-native-client/connection/request.cc
index 50ea029..d4a4603 100644
--- a/hbase-native-client/connection/request.cc
+++ b/hbase-native-client/connection/request.cc
@@ -30,16 +30,16 @@ Request::Request(std::shared_ptr 
req,
 
 std::unique_ptr Request::get() {
   return std::make_unique(std::make_shared(),
-  std::make_shared(),
-  "Get");
+   std::make_shared(),
+   "Get");
 }
 std::unique_ptr Request::mutate() {
-  return 
std::make_unique(std::make_shared(),
-  
std::make_shared(),
-  "Mutate");
+  return std::make_unique(
+  std::make_shared(),
+  std::make_shared(), "Mutate");
 }
 std::unique_ptr Request::scan() {
   return std::make_unique(std::make_shared(),
-  std::make_shared(),
-  "Scan");
+   std::make_shared(),
+   "Scan");
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/BUCK
--
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
index 2b65b12..ad12f7d 100644
--- a/hbase-native-client/utils/BUCK
+++ b/hbase-native-client/utils/BUCK
@@ -23,18 +23,18 @@ cxx_library(name="utils",
 "user-util.cc",
 ],
 deps=[
-  '//third-party:folly',
+'//third-party:folly',
 ],
 visibility=[
 'PUBLIC',
 ],
 tests=[
-  ":user-util-test"
-],)
+":user-util-test"
+], )
 cxx_test(name="user-util-test",
  srcs=[
  "user-util-test.cc",
  ],
  deps=[
  ":utils",
- ],)
+ ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util-test.cc
--
diff --git a/hbase-native-client/utils/user-util-test.cc 
b/hbase-native-client/utils/user-util-test.cc
index 1e72bcc..7c11d8c 100644
--- a/hbase-native-client/utils/user-util-test.cc
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -17,9 +17,9 @@
  *
  */
 
+#include 
 #include 
 #include 
-#include 
 
 #include "utils/user-util.h"
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util.cc
--
diff --git a/hbase-native-client/utils/user-util.cc 
b/hbase-native-client/utils/user-util.cc
index c4427e3..b801537 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -19,10 +19,10 @@
 
 #include "utils/user-util.h"
 
+#include 
 #include 
-#include 
 #include 
-#include 
+#include 
 
 using 

[48/50] [abbrv] hbase git commit: HBASE-15770 Stop using wangle's global executor

[eclark]

---
diff --git a/hbase-native-client/core/location-cache-test.cc 
b/hbase-native-client/core/location-cache-test.cc
index 172799d..8bc6383 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -18,14 +18,15 @@
  */
 #include 
 #include 
-#include 
 
 #include "location-cache.h"
 using namespace hbase;
 
 TEST(LocationCacheTest, TestGetMetaNodeContents) {
   // TODO(elliott): need to make a test utility for this.
-  LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+  auto cpu = std::make_shared(4);
+  auto io = std::make_shared(4);
+  LocationCache cache{"localhost:2181", cpu, io};
   auto f = cache.LocateMeta();
   auto result = f.get();
   ASSERT_FALSE(f.hasException());

http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index 583d305..6c018f9 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -20,7 +20,8 @@
 
 #include 
 #include 
-#include 
+#include 
+#include 
 
 #include "connection/response.h"
 #include "if/Client.pb.h"
@@ -48,10 +49,13 @@ using hbase::pb::RegionInfo;
 // TODO(eclark): make this configurable on client creation
 static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
 
-LocationCache::LocationCache(string quorum_spec,
- shared_ptr executor)
-: quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
-  meta_lock_(), cp_(), meta_util_(), zk_(nullptr) {
+LocationCache::LocationCache(
+std::string quorum_spec,
+std::shared_ptr cpu_executor,
+std::shared_ptr io_executor)
+: quorum_spec_(quorum_spec), cpu_executor_(cpu_executor),
+  meta_promise_(nullptr), meta_lock_(), cp_(io_executor), meta_util_(),
+  zk_(nullptr) {
   zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
 }
 
@@ -79,7 +83,7 @@ void LocationCache::InvalidateMeta() {
 /// MUST hold the meta_lock_
 void LocationCache::RefreshMetaLocation() {
   meta_promise_ = make_unique<SharedPromise>();
-  executor_->add([&] {
+  cpu_executor_->add([&] {
 meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
   });
 }
@@ -109,10 +113,9 @@ ServerName LocationCache::ReadMetaLocation() {
 
 Future<std::shared_ptr>
 LocationCache::LocateFromMeta(const TableName , const string ) {
-  auto exec = wangle::getCPUExecutor();
   return this->LocateMeta()
-  .via(exec.get())
-  .then([ exec = exec, this ](ServerName sn) { return this->cp_.get(sn); })
+  .via(cpu_executor_.get())
+  .then([this](ServerName sn) { return this->cp_.get(sn); })
   .then([&](std::shared_ptr service) {
 return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
   })
@@ -121,7 +124,7 @@ LocationCache::LocateFromMeta(const TableName , const 
string ) {
 // a region location.
 return this->CreateLocation(std::move(resp));
   })
-  .then([ exec = exec, this ](std::shared_ptr rl) {
+  .then([this](std::shared_ptr rl) {
 // Now fill out the connection.
 rl->set_service(cp_.get(rl->server_name()));
 return rl;
@@ -129,14 +132,14 @@ LocationCache::LocateFromMeta(const TableName , const 
string ) {
 }
 
 /**
- * Filter to remove a service from the location cache and the connection cache 
on errors
+ * Filter to remove a service from the location cache and the connection cache
+ * on errors
  * or on cloase.
  */
 class RemoveServiceFilter
 : public ServiceFilter<std::unique_ptr, Response> {
 
 public:
-
   /** Create a new filter. */
   RemoveServiceFilter(std::shared_ptr service, ServerName sn,
   ConnectionPool )
@@ -157,7 +160,6 @@ public:
 }
   }
 
-
   /** Has this been closed */
   virtual bool isAvailable() override {
 return !released && service_->isAvailable();

http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/location-cache.h
--
diff --git a/hbase-native-client/core/location-cache.h 
b/hbase-native-client/core/location-cache.h
index 88bec18..e077750 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -21,6 +21,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 #include 
 
 #include 
@@ -51,7 +53,8 @@ public:
* @param executor The cpu executor to run on.
*/
   LocationCache(std::string quorum_spec,
-std::shared_ptr executor);
+std::shared_ptr cpu_exector,
+std::shared_p

[27/50] [abbrv] hbase git commit: HBASE-15761 Add on more server name tests

[eclark]

HBASE-15761 Add on more server name tests


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/297a4dff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/297a4dff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/297a4dff

Branch: refs/heads/HBASE-14850
Commit: 297a4dff638649f346e83c8cc685f0009fb3
Parents: fdf0023
Author: Elliott Clark 
Authored: Wed May 4 12:04:18 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/serde/server-name-test.cc | 18 ++
 hbase-native-client/serde/server-name.h   |  4 +++-
 2 files changed, 21 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/297a4dff/hbase-native-client/serde/server-name-test.cc
--
diff --git a/hbase-native-client/serde/server-name-test.cc 
b/hbase-native-client/serde/server-name-test.cc
index 35dcbc1..2281fa2 100644
--- a/hbase-native-client/serde/server-name-test.cc
+++ b/hbase-native-client/serde/server-name-test.cc
@@ -30,3 +30,21 @@ TEST(TestServerName, TestMakeServerName) {
   ASSERT_EQ("test", sn.host_name());
   ASSERT_EQ(123, sn.port());
 }
+
+TEST(TestServerName, TestIps) {
+  auto sn = folly::to("127.0.0.1:999");
+  ASSERT_EQ("127.0.0.1", sn.host_name());
+  ASSERT_EQ(999, sn.port());
+}
+
+TEST(TestServerName, TestThrow) {
+  ASSERT_ANY_THROW(folly::to("Ther's no colon here"));
+}
+
+TEST(TestServerName, TestIPV6) {
+  auto sn = folly::to("[1]:123");
+
+  ASSERT_EQ("[1]", sn.host_name());
+  ASSERT_EQ(123, sn.port());
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/297a4dff/hbase-native-client/serde/server-name.h
--
diff --git a/hbase-native-client/serde/server-name.h 
b/hbase-native-client/serde/server-name.h
index bdba087..9844465 100644
--- a/hbase-native-client/serde/server-name.h
+++ b/hbase-native-client/serde/server-name.h
@@ -12,7 +12,9 @@ template  void parseTo(String in, ServerName 
) {
   std::string s = folly::to(in);
 
   auto delim = s.rfind(":");
-  DCHECK(delim != std::string::npos);
+  if (delim == std::string::npos) {
+throw std::runtime_error("Couldn't parse server name");
+  }
   out.set_host_name(s.substr(0, delim));
   // Now keep everything after the : (delim + 1) to the end.
   out.set_port(folly::to(s.substr(delim + 1)));

[35/50] [abbrv] hbase git commit: HBASE-15851 Makefile update for build env

[eclark]

HBASE-15851 Makefile update for build env

1) Makefile to compile protobuf sources which are extracted in build
2) Added -O2 and -D_GLIBCXX_USE_CXX11_ABI=0 compilation flags
3) Header files added in Makefile


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df14a6a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df14a6a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df14a6a7

Branch: refs/heads/HBASE-14850
Commit: df14a6a7a680dfd15642ab5c420b5da2c8153f88
Parents: 8014cdf
Author: sudeeps 
Authored: Sun May 22 09:38:47 2016 +1000
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Makefile  | 132 +++--
 hbase-native-client/Makefile.protos   |  53 
 hbase-native-client/core/meta-utils.h |   2 +-
 3 files changed, 159 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/df14a6a7/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 826233f..7e68b6a 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,37 +1,115 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-build:
-   $(shell buck build core/... )
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+ 
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+PROTO_SRC_DIR = build/if
+MODULES = connection core serde test-util utils
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = . build/
 
-check:
-   $(shell buck test --all --no-results-cache )
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 
-pedantic -fPIC
+LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
+LINKFLAG = -shared
 
-doc:
-   $(shell doxygen hbase.doxygen > /dev/null )
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard 
if/*.proto))
+DEPS =  $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard 
if/*.proto))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+   
+LIB_DIR = /usr/local
+LIB_LIBDIR = $(LIB_DIR)/lib
+LIB_INCDIR = $(LIB_DIR)/include
+LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so
+ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a
+LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so
+ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
+
+vpath %.cc $(SRC_DIR)
+
+$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+   $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(LIB_RELEASE): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+   $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES) 
+endef
+
+.PHONY: all clean install 
+
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) 
$(ARC_RELEASE) 
+
+checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+
+protos: createprotosrc
+   @make all -f Makefile.protos
+
+createprotosrc:$(PROTO_SRC_DIR)
+   @protoc --proto_path=if --cpp_out=$(PROTO_SRC_DIR) if/*.proto
+
+install:
+   cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
+   cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
+   cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
+   cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+   
+$(PROTO_SRC_DIR):
+   @mkdir -p $@
+
+$(DEBUG_BUILD_DIR):
+   @mkdir -p $@

[45/50] [abbrv] hbase git commit: HBASE-15945 Patch for Cell

[eclark]

HBASE-15945 Patch for Cell

This patch consists of Cell implementation without additional interfaces.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa3ab420
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa3ab420
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa3ab420

Branch: refs/heads/HBASE-14850
Commit: fa3ab420d19bba853d96576176257b5c12f1a48f
Parents: 2b6341d
Author: Sudeep Sunthankar 
Authored: Mon Jul 4 21:02:25 2016 +1000
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/core/cell-test.cc | 175 +
 hbase-native-client/core/cell.cc  |  77 +
 hbase-native-client/core/cell.h   |  60 ++
 3 files changed, 312 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fa3ab420/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
new file mode 100644
index 000..cbe50eb
--- /dev/null
+++ b/hbase-native-client/core/cell-test.cc
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+
+#include 
+#include 
+#include 
+
+using namespace hbase;
+TEST(CellTest, CellFailureTest) {
+  CellType cell_type = CellType::PUT;
+  std::string row = "row";
+  std::string family = "family";
+  std::string column = "column";
+  std::string value = "value";
+  long timestamp = std::numeric_limits::max();
+  std::string tags = "";
+  std::unique_ptr cell(
+  new Cell(row, family, column, timestamp, value, cell_type));
+  if (cell.get()) {
+EXPECT_NE("row-value", cell.get()->Row());
+EXPECT_NE("family-value", cell.get()->Family());
+EXPECT_NE("column-value", cell.get()->Qualifier());
+EXPECT_NE("value-value", cell.get()->Value());
+EXPECT_NE(8975431260, cell.get()->Timestamp());
+EXPECT_NE(CellType::MAXIMUM, cell.get()->Type());
+  }
+}
+
+TEST(CellTest, CellSuceessTest) {
+  std::string row = "row-value";
+  std::string family = "family-value";
+  std::string column = "column-value";
+  std::string value = "value-value";
+  long timestamp = std::numeric_limits::max();
+  CellType cell_type = CellType::PUT;
+  const std::unique_ptr cell(
+  new Cell(row, family, column, timestamp, value, cell_type));
+  if (cell.get()) {
+EXPECT_EQ(row, cell.get()->Row());
+EXPECT_EQ(family, cell.get()->Family());
+EXPECT_EQ(column, cell.get()->Qualifier());
+EXPECT_EQ(value, cell.get()->Value());
+EXPECT_EQ(timestamp, cell.get()->Timestamp());
+EXPECT_EQ(cell_type, cell.get()->Type());
+  }
+}
+
+TEST(CellTest, MultipleCellsTest) {
+  std::vector cells;
+  for (int i = 0; i < 5; i++) {
+std::string row = "row-value";
+std::string family = "family-value";
+std::string column = "column-value";
+std::string value = "value-value";
+long timestamp = std::numeric_limits::max();
+row += std::to_string(i);
+value += std::to_string(i);
+CellType cell_type = CellType::PUT;
+const Cell *cell = new Cell(row, family, column, timestamp, value,
+cell_type);
+cells.push_back(cell);
+  }
+  int i = 0;
+  for (const auto cell : cells) {
+std::string row = "row-value";
+std::string value = "value-value";
+row += std::to_string(i);
+value += std::to_string(i);
+EXPECT_EQ(row, cell->Row());
+EXPECT_EQ("family-value", cell->Family());
+EXPECT_EQ("column-value", cell->Qualifier());
+EXPECT_EQ(value, cell->Value());
+EXPECT_EQ(std::numeric_limits::max(), cell->Timestamp());
+EXPECT_EQ(CellType::PUT, cell->Type());
+i += 1;
+  }
+  for (const auto cell : cells) {
+delete cell;
+  }
+  cells.clear();
+}
+
+TEST(CellTest, CellRowTest) {
+  std::string row = "only-row";
+  std::string family = "";
+  std::string column = 

[29/50] [abbrv] hbase git commit: HBASE-15620 Add on Call serialization

[eclark]

', ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-dispatcher.cc
--
diff --git a/hbase-native-client/connection/client-dispatcher.cc 
b/hbase-native-client/connection/client-dispatcher.cc
new file mode 100644
index 000..25cff7d
--- /dev/null
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "connection/client-dispatcher.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+void ClientDispatcher::read(Context *ctx, Response in) {
+  auto call_id = in.call_id();
+  auto search = requests_.find(call_id);
+  CHECK(search != requests_.end());
+  auto p = std::move(search->second);
+
+  requests_.erase(call_id);
+
+  // TODO(eclark): check if the response
+  // is an exception. If it is then set that.
+  p.setValue(in);
+}
+
+Future ClientDispatcher::operator()(Request arg) {
+  auto call_id = ++current_call_id_;
+
+  arg.set_call_id(call_id);
+  auto  = requests_[call_id];
+  auto f = p.getFuture();
+  p.setInterruptHandler([call_id, this](const folly::exception_wrapper ) {
+this->requests_.erase(call_id);
+  });
+  this->pipeline_->write(arg);
+
+  return f;
+}
+
+Future ClientDispatcher::close() { return ClientDispatcherBase::close(); 
}
+
+Future ClientDispatcher::close(Context *ctx) {
+  return ClientDispatcherBase::close(ctx);
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
new file mode 100644
index 000..c3987c9
--- /dev/null
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include 
+
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
+
+namespace hbase {
+class ClientDispatcher
+: public wangle::ClientDispatcherBase<SerializePipeline, Request,
+  Response> {
+public:
+  void read(Context *ctx, Response in) override;
+  folly::Future operator()(Request arg) override;
+  folly::Future close(Context *ctx) override;
+  folly::Future close() override;
+
+private:
+  std::unordered_map<uint32_t, folly::Promise> requests_;
+  uint32_t current_call_id_ = 10;
+};
+} // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
new file mode 100644
index 000..74b23ef
--- /dev/null
+++ b/hbase-native-client/connection/client-handler.cc
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * &qu

[47/50] [abbrv] hbase git commit: HBASE-15826 Clean up ASF license issues

[eclark]

HBASE-15826 Clean up ASF license issues


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e718c7dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e718c7dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e718c7dc

Branch: refs/heads/HBASE-14850
Commit: e718c7dc9a4c80015ef7a005714322a2cbb9f6ea
Parents: f6ea493
Author: Elliott Clark 
Authored: Fri May 13 14:53:47 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Makefile | 17 +
 hbase-native-client/bin/copy-protobuf.sh | 16 
 hbase-native-client/bin/format-code.sh   | 16 
 hbase-native-client/bin/start-local-hbase.sh |  1 -
 hbase-native-client/hbase.doxygen| 22 +++---
 hbase-native-client/serde/server-name.h  | 19 +++
 6 files changed, 87 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 90ec6ea..826233f 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 build:
$(shell buck build core/... )
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/copy-protobuf.sh
--
diff --git a/hbase-native-client/bin/copy-protobuf.sh 
b/hbase-native-client/bin/copy-protobuf.sh
index c9e70f2..36541cf 100755
--- a/hbase-native-client/bin/copy-protobuf.sh
+++ b/hbase-native-client/bin/copy-protobuf.sh
@@ -1,4 +1,20 @@
 #!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 set -euo pipefail
 IFS=$'\n\t'
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/format-code.sh
--
diff --git a/hbase-native-client/bin/format-code.sh 
b/hbase-native-client/bin/format-code.sh
index 24d6515..36a114d 100755
--- a/hbase-native-client/bin/format-code.sh
+++ b/hbase-native-client/bin/format-code.sh
@@ -1,4 +1,20 @@
 #!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 set -euo pipefail
 IFS=$'\n\t'
 

[33/50] [abbrv] hbase git commit: HBASE-15687 Allow decoding more than GetResponse from the server

[eclark]

tion/client-handler.cc
@@ -47,14 +47,30 @@ void ClientHandler::read(Context *ctx, 
std::unique_ptr buf) {
 LOG(INFO) << "Read ResponseHeader size=" << used_bytes
   << " call_id=" << header.call_id()
   << " has_exception=" << header.has_exception();
+
+// Get the response protobuf from the map
+auto search = resp_msgs_.find(header.call_id());
+// It's an error if it's not there.
+CHECK(search != resp_msgs_.end());
+auto resp_msg = search->second;
+CHECK(resp_msg != nullptr);
+
+// Make sure we don't leak the protobuf
+resp_msgs_.erase(search);
+
+// set the call_id.
+// This will be used to by the dispatcher to match up
+// the promise with the response.
 received.set_call_id(header.call_id());
 
+// If there was an exception then there's no
+// data left on the wire.
 if (header.has_exception() == false) {
   buf->trimStart(used_bytes);
-  // For now assume that everything was a get.
-  // We'll need to set this up later.
-  received.set_response(std::make_shared());
-  used_bytes = deser_.parse_delimited(buf.get(), 
received.response().get());
+  used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
+  // Make sure that bytes were parsed.
+  CHECK(used_bytes == buf->length());
+  received.set_response(resp_msg);
 }
 ctx->fireRead(std::move(received));
   }
@@ -62,7 +78,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr 
buf) {
 
 // TODO(eclark): Figure out how to handle the
 // network errors that are going to come.
-Future ClientHandler::write(Context *ctx, Request r) {
+Future ClientHandler::write(Context *ctx, std::unique_ptr r) {
   // Keep track of if we have sent the header.
   if (UNLIKELY(need_send_header_)) {
 need_send_header_ = false;
@@ -78,5 +94,7 @@ Future ClientHandler::write(Context *ctx, Request r) {
 ctx->fireWrite(std::move(pre));
   }
 
-  return ctx->fireWrite(ser_.request(r.call_id(), r.method(), r.msg()));
+  resp_msgs_[r->call_id()] = r->resp_msg();
+  return ctx->fireWrite(
+  ser_.request(r->call_id(), r->method(), r->req_msg().get()));
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-handler.h
--
diff --git a/hbase-native-client/connection/client-handler.h 
b/hbase-native-client/connection/client-handler.h
index dbaf5a0..41bb883 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -30,20 +30,30 @@ namespace hbase {
 class Request;
 class Response;
 }
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
 
 namespace hbase {
-class ClientHandler
-: public wangle::Handler<std::unique_ptr, Response, Request,
- std::unique_ptr> {
+class ClientHandler : public wangle::Handler<std::unique_ptr,
+ Response, 
std::unique_ptr,
+ std::unique_ptr> {
 public:
   ClientHandler(std::string user_name);
   void read(Context *ctx, std::unique_ptr msg) override;
-  folly::Future write(Context *ctx, Request r) override;
+  folly::Future write(Context *ctx,
+   std::unique_ptr r) override;
 
 private:
   bool need_send_header_ = true;
   std::string user_name_;
   ClientSerializer ser_;
   ClientDeserializer deser_;
+
+  // in flight requests
+  std::unordered_map<uint32_t, std::shared_ptr>
+  resp_msgs_;
 };
 } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/connection-factory.cc
--
diff --git a/hbase-native-client/connection/connection-factory.cc 
b/hbase-native-client/connection/connection-factory.cc
index 5d1b0da..7073f9d 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -45,14 +45,14 @@ ConnectionFactory::ConnectionFactory() {
   bootstrap_.pipelineFactory(std::make_shared());
 }
 
-std::shared_ptr<Service<Request, Response>>
+std::shared_ptr<Service<std::unique_ptr, Response>>
 ConnectionFactory::make_connection(std::string host, int port) {
   // Connect to a given server
   // Then when connected create a ClientDispactcher.
   auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
   auto dispatcher = std::make_shared();
   dispatcher->setPipeline(pipeline);
-  auto service =
-  std::make_shared<CloseOnReleaseFilter<Request, Response>>(dispatcher);
+  auto service = std::make_shared<
+  CloseOnReleaseFilter<std::unique_ptr, Response>>(dispatcher);
   return service;
 }

http://git-wip-us.

[43/50] [abbrv] hbase git commit: HBASE-15731 Add on a connection pool

[eclark]

ied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/get-result.h"

http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/get-result.h
--
diff --git a/hbase-native-client/core/get-result.h 
b/hbase-native-client/core/get-result.h
deleted file mode 100644
index a49ad98..000
--- a/hbase-native-client/core/get-result.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include 
-
-namespace hbase {
-
-class GetResult {
-public:
-  explicit GetResult(std::string key);
-
-private:
-  std::string key_;
-};
-} // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index 5925f4a..c81deba 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -77,7 +77,6 @@ ServerName LocationCache::ReadMetaLocation() {
   int zk_result =
   zoo_get(this->zk_, META_ZNODE_NAME, 0,
   reinterpret_cast(buf->writableData()), , nullptr);
-  LOG(ERROR) << "len = " << len;
   if (zk_result != ZOK || len < 9) {
 LOG(ERROR) << "Error getting meta location.";
 throw runtime_error("Error getting meta location");

http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/simple-client.cc
--
diff --git a/hbase-native-client/core/simple-client.cc 
b/hbase-native-client/core/simple-client.cc
index 2cb6200..11dcd68 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -25,7 +25,7 @@
 #include 
 #include 
 
-#include "connection/connection-factory.h"
+#include "connection/connection-pool.h"
 #include "core/client.h"
 #include "if/Client.pb.h"
 #include "if/ZooKeeper.pb.h"
@@ -33,13 +33,20 @@
 using namespace folly;
 using namespace std;
 using namespace std::chrono;
-using namespace hbase;
-using namespace hbase::pb;
-using namespace google::protobuf;
+using hbase::Response;
+using hbase::Request;
+using hbase::HBaseService;
+using hbase::LocationCache;
+using hbase::ConnectionPool;
+using hbase::pb::ServerName;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+using hbase::pb::GetRequest;
+using hbase::pb::GetResponse;
 
 // TODO(eclark): remove the need for this.
 DEFINE_string(region, "1588230740", "What region to send a get to");
 DEFINE_string(row, "test", "What row to get");
+DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
 
 int main(int argc, char *argv[]) {
   google::SetUsageMessage(
@@ -48,46 +55,52 @@ int main(int argc, char *argv[]) {
   google::InitGoogleLogging(argv[0]);
 
   // Create a connection factory
-  ConnectionFactory cf;
-
-  LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
-
-  auto result = cache.LocateMeta().get();
-
-  // Create a connection to the local host
-  auto conn = cf.make_connection(result.host_name(), result.port());
-
-  // Send the request
-  auto r = Request::get();
-
-  // This is a get request so make that
-  auto req_msg = static_pointer_cast(r->req_msg());
-
-  // Set what region
-  req_msg->mutable_region()->set_value(FLAGS_region);
-  // It's always this.
-  req_msg->mutable_region()->set_type(
-  RegionSpecifier_RegionSpecifierType::
-  RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
-
-  // What row.
-  req_msg->mutable_get()->set_row(FLAGS_row);
-
-  // Send it.
-  auto resp = (*conn)(std::move(r)).get(milliseconds(5000));
-
-  auto get_resp = std::static_pointer_cast(resp.response());
-  cout << "GetResponse has_result = " << get_

[41/50] [abbrv] hbase git commit: HBASE-15690 Add utility to get current username

[eclark]

HBASE-15690 Add utility to get current username

Summary:
Add a class to fine the username of the current process.
It will only call out once and is multithread safe

Test Plan: Unit Test

Differential Revision: https://reviews.facebook.net/D57081


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b2d65f1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b2d65f1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b2d65f1

Branch: refs/heads/HBASE-14850
Commit: 8b2d65f1aa7c7dddcde4be03133f8c35e1fbcba2
Parents: b23e863
Author: Elliott Clark 
Authored: Thu Apr 21 21:17:11 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/utils/BUCK  | 40 +
 hbase-native-client/utils/user-util-test.cc | 34 +++
 hbase-native-client/utils/user-util.cc  | 55 
 hbase-native-client/utils/user-util.h   | 37 
 4 files changed, 166 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/BUCK
--
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
new file mode 100644
index 000..2b65b12
--- /dev/null
+++ b/hbase-native-client/utils/BUCK
@@ -0,0 +1,40 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="utils",
+exported_headers=[
+"user-util.h",
+],
+srcs=[
+"user-util.cc",
+],
+deps=[
+  '//third-party:folly',
+],
+visibility=[
+'PUBLIC',
+],
+tests=[
+  ":user-util-test"
+],)
+cxx_test(name="user-util-test",
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)

http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util-test.cc
--
diff --git a/hbase-native-client/utils/user-util-test.cc 
b/hbase-native-client/utils/user-util-test.cc
new file mode 100644
index 000..2a7434f
--- /dev/null
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include 
+#include 
+#include 
+
+#include "utils/user-util.h"
+
+using namespace std;
+using namespace hbase;
+
+TEST(TestUserUtil, TestGetSomething) {
+  UserUtil u_util;
+  string name = u_util.user_name();
+
+  ASSERT_GT(name.length(), 0);
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util.cc
--
diff --git a/hbase-native-client/utils/user-util.cc 
b/hbase-native-client/utils/user-util.cc
new file mode 100644
index 000..c4427e3
--- /dev/null
+++ b/hbase-native-client/utils/user-util.cc
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * 

[40/50] [abbrv] hbase git commit: HBASE-15655 Set up doxygen for documentation

[eclark]

HBASE-15655 Set up doxygen for documentation

Summary:
Add on doxygen config.
It outputs to the docs folder.
Added on a makefile

Test Plan:
doxygen hbase.doxygen

Differential Revision: https://reviews.facebook.net/D57663


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f034294f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f034294f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f034294f

Branch: refs/heads/HBASE-14850
Commit: f034294f3a72bcb6c11330d6fa1f9b6c028f1f1c
Parents: 190fc7a
Author: Elliott Clark 
Authored: Wed May 4 18:19:33 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .gitignore|   1 +
 hbase-native-client/Dockerfile|   2 +-
 hbase-native-client/Makefile  |  20 +++
 hbase-native-client/hbase.doxygen | 319 +
 4 files changed, 341 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/.gitignore
--
diff --git a/.gitignore b/.gitignore
index 626a4d0..1b1e9d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ hbase-*/test
 *.ipr
 patchprocess/
 dependency-reduced-pom.xml
+docs/

http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 8e33e74..be0cbc6 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -24,7 +24,7 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g 
-fno-omit-frame-pointer -O2 -p
 
 ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
 
-RUN apt-get install -y vim maven inetutils-ping python-pip && \
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && 
\
   pip install yapf && \
   apt-get -qq clean && \
   apt-get -y -qq autoremove && \

http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
new file mode 100644
index 000..90ec6ea
--- /dev/null
+++ b/hbase-native-client/Makefile
@@ -0,0 +1,20 @@
+build:
+   $(shell buck build core/... )
+
+check:
+   $(shell buck test --all --no-results-cache )
+
+doc:
+   $(shell doxygen hbase.doxygen > /dev/null )
+
+clean:
+   $(shell rm -rf docs )
+
+help:
+   @echo "Available targets:"
+   @echo ""
+   @echo " build : will build everything."
+   @echo " clean : will remove the docs folder"
+   @echo " check : will test everything."
+
+all: build doc check

http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/hbase.doxygen
--
diff --git a/hbase-native-client/hbase.doxygen 
b/hbase-native-client/hbase.doxygen
new file mode 100644
index 000..22e2d21
--- /dev/null
+++ b/hbase-native-client/hbase.doxygen
@@ -0,0 +1,319 @@
+# Doxyfile 1.8.11
+
+#---
+# Project related configuration options
+#---
+DOXYFILE_ENCODING  = UTF-8
+PROJECT_NAME   = "HBase Native Client"
+PROJECT_NUMBER =
+PROJECT_BRIEF  =
+PROJECT_LOGO   = 
../hbase-server/src/main/resources/hbase-webapps/static/hbase_logo_small.png
+OUTPUT_DIRECTORY   = docs
+CREATE_SUBDIRS = YES
+ALLOW_UNICODE_NAMES= NO
+OUTPUT_LANGUAGE= English
+BRIEF_MEMBER_DESC  = YES
+REPEAT_BRIEF   = YES
+ABBREVIATE_BRIEF   =
+ALWAYS_DETAILED_SEC= NO
+INLINE_INHERITED_MEMB  = NO
+FULL_PATH_NAMES= NO
+STRIP_FROM_PATH=
+STRIP_FROM_INC_PATH=
+SHORT_NAMES= NO
+JAVADOC_AUTOBRIEF  = NO
+QT_AUTOBRIEF   = NO
+MULTILINE_CPP_IS_BRIEF = NO
+INHERIT_DOCS   = YES
+SEPARATE_MEMBER_PAGES  = NO
+TAB_SIZE   = 4
+ALIASES=
+TCL_SUBST  =
+OPTIMIZE_OUTPUT_FOR_C  = NO
+OPTIMIZE_OUTPUT_JAVA   = NO
+OPTIMIZE_FOR_FORTRAN   = NO
+OPTIMIZE_OUTPUT_VHDL   = NO
+EXTENSION_MAPPING  =
+MARKDOWN_SUPPORT   = YES
+AUTOLINK_SUPPORT   = YES
+BUILTIN_STL_SUPPORT= NO
+CPP_CLI_SUPPORT= NO
+SIP_SUPPORT= NO
+IDL_PROPERTY_SUPPORT   = YES
+DISTRIBUTE_GROUP_DOC   = NO
+GROUP_NESTED_COMPOUNDS = NO
+SUBGROUPING= YES
+INLINE_GROUPED_CLASSES = NO
+INLINE_SIMPLE_STRUCTS  = NO
+TYPEDEF_HIDES_STRUCT   = NO
+LOOKUP_CACHE_SIZE  = 0

[26/50] [abbrv] hbase git commit: HBASE-14855 Connect to regionserver

[eclark]

ion-cache.cc",
 "mutation.cc",
+"pipeline.cc",
 "put.cc",
-"delete.cc",
 "scanner.cc",
-"location-cache.cc",
+"table-name.cc",
 ],
 deps=[
 "//if:if",
-"//third-party:zookeeper_mt",
 "//third-party:folly",
 "//third-party:wangle",
+"//third-party:zookeeper_mt",
 ],
 visibility=[
 'PUBLIC',
@@ -68,3 +85,6 @@ cxx_test(name="location-cache-test",
  ":core",
  ],
  run_test_separately=True, )
+cxx_binary(name="simple-client",
+   srcs=["simple-client.cc", ],
+   deps=[":core", ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client-dispatcher.cc
--
diff --git a/hbase-native-client/core/client-dispatcher.cc 
b/hbase-native-client/core/client-dispatcher.cc
new file mode 100644
index 000..d356759
--- /dev/null
+++ b/hbase-native-client/core/client-dispatcher.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/client-dispatcher.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+void ClientDispatcher::read(Context *ctx, Response in) {
+  auto call_id = in.call_id();
+  auto search = requests_.find(call_id);
+  CHECK(search != requests_.end());
+  auto p = std::move(search->second);
+  requests_.erase(call_id);
+
+  // TODO(eclark): check if the response
+  // is an exception. If it is then set that.
+  p.setValue(in);
+}
+
+Future ClientDispatcher::operator()(Request arg) {
+  auto call_id = ++current_call_id_;
+  arg.set_call_id(call_id);
+  auto  = requests_[call_id];
+  auto f = p.getFuture();
+  p.setInterruptHandler([call_id, this](const folly::exception_wrapper ) {
+this->requests_.erase(call_id);
+  });
+  this->pipeline_->write(arg);
+
+  return f;
+}
+
+Future ClientDispatcher::close() { return ClientDispatcherBase::close(); 
}
+
+Future ClientDispatcher::close(Context *ctx) {
+  return ClientDispatcherBase::close(ctx);
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client-dispatcher.h
--
diff --git a/hbase-native-client/core/client-dispatcher.h 
b/hbase-native-client/core/client-dispatcher.h
new file mode 100644
index 000..4b9d35a
--- /dev/null
+++ b/hbase-native-client/core/client-dispatcher.h
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include 
+
+#include "core/pipeline.h"
+#include "core/request.h"
+#include "core/response.h"
+
+namespace hbase {
+class ClientDispatcher
+: public wangle::ClientDispatcherBase<SerializePipeline, Request,
+  Response> {
+public:
+  void read(Context *ctx, Response in) override;
+  folly::Future operator()(Request arg) override;
+  folly::Future close(Context *ctx) override;
+  folly::Future close() override;
+
+private:
+  std

[39/50] [abbrv] hbase git commit: HBASE-15777 Fix needs header in client handler

[eclark]

HBASE-15777 Fix needs header in client handler


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/16e1de78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/16e1de78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/16e1de78

Branch: refs/heads/HBASE-14850
Commit: 16e1de78a6a0fa92adbd330890f213d8763adb83
Parents: 79b5085
Author: Elliott Clark <ecl...@apache.org>
Authored: Thu May 5 13:51:18 2016 -0700
Committer: Elliott Clark <ecl...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-handler.cc| 31 
 hbase-native-client/connection/client-handler.h | 19 +++-
 2 files changed, 36 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/16e1de78/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index b92ad89..4fdb7ae 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
 using google::protobuf::Message;
 
 ClientHandler::ClientHandler(std::string user_name)
-: user_name_(user_name), need_send_header_(true), serde_(),
+: user_name_(user_name), serde_(), header_info_(),
   resp_msgs_(
   make_unique<folly::AtomicHashMap<
   uint32_t, std::shared_ptr>>(5000)) {}
@@ -81,22 +81,27 @@ void ClientHandler::read(Context *ctx, 
std::unique_ptr buf) {
   }
 }
 
-// TODO(eclark): Figure out how to handle the
-// network errors that are going to come.
 Future ClientHandler::write(Context *ctx, std::unique_ptr r) {
   // Keep track of if we have sent the header.
-  if (UNLIKELY(need_send_header_)) {
-need_send_header_ = false;
+  //
+  // even though the bool is atomic we can load it lazily here.
+  if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
 
-// Should we be sending just one fireWrite?
-// Right now we're sending one for the header
-// and one for the request.
+// Grab the lock.
+// We need to make sure that no one gets past here without there being a
+// hearder sent.
+std::lock_guard lock(header_info_->mutex_);
+
+// Now see if we are the first thread to get through.
 //
-// That doesn't seem like too bad, but who knows.
-auto pre = serde_.Preamble();
-auto header = serde_.Header(user_name_);
-pre->appendChain(std::move(header));
-ctx->fireWrite(std::move(pre));
+// If this is the first thread to get through then the
+// need_send_header will have been true before this.
+if (header_info_->need_.exchange(false)) {
+  auto pre = serde_.Preamble();
+  auto header = serde_.Header(user_name_);
+  pre->appendChain(std::move(header));
+  ctx->fireWrite(std::move(pre));
+}
   }
 
   resp_msgs_->insert(r->call_id(), r->resp_msg());

http://git-wip-us.apache.org/repos/asf/hbase/blob/16e1de78/hbase-native-client/connection/client-handler.h
--
diff --git a/hbase-native-client/connection/client-handler.h 
b/hbase-native-client/connection/client-handler.h
index be5143c..1a4275f 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -21,6 +21,8 @@
 #include 
 #include 
 
+#include 
+#include 
 #include 
 
 #include "serde/rpc.h"
@@ -29,6 +31,7 @@
 namespace hbase {
 class Request;
 class Response;
+class HeaderInfo;
 }
 namespace google {
 namespace protobuf {
@@ -37,6 +40,7 @@ class Message;
 }
 
 namespace hbase {
+
 class ClientHandler : public wangle::Handler<std::unique_ptr,
  Response, 
std::unique_ptr,
  std::unique_ptr> {
@@ -47,7 +51,7 @@ public:
std::unique_ptr r) override;
 
 private:
-  bool need_send_header_;
+  std::unique_ptr header_info_;
   std::string user_name_;
   RpcSerde serde_;
 
@@ -56,4 +60,17 @@ private:
   uint32_t, std::shared_ptr>>
   resp_msgs_;
 };
+
+/**
+ * Class to contain the info about if the connection header and preamble has
+ * been sent.
+ *
+ * We use a serperate class here so that ClientHandler is relocatable.
+ */
+class HeaderInfo {
+public:
+  HeaderInfo() : need_(true), mutex_() {}
+  std::atomic need_;
+  std::mutex mutex_;
+};
 } // namespace hbase

[04/50] [abbrv] hbase git commit: HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove TimeRangeTracker as point of contention when many threads reading a StoreFile Fixes HBASE-16074 ITBLL fails,

[eclark]

HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove
 TimeRangeTracker as point of contention when many threads reading a StoreFile
 Fixes HBASE-16074 ITBLL fails, reports lost big or tiny families broken
 scanning because of a side effect of a clean up in  HBASE-15650 to make
 TimeRange construction consistent exposed a latent issue in
 TimeRange#compare. See HBASE-16074 for more detail.

Also change HFile Writer constructor so we pass in the TimeRangeTracker, if one,
on construction rather than set later (the flag and reference were not volatile
so could have made for issues in concurrent case). And make sure the 
construction
of a TimeRange from a TimeRangeTracer on open of an HFile Reader never makes a
bad minimum value, one that would preclude us reading any values from a file
(set min to 0)

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
 Call through to next constructor (if minStamp was 0, we'd skip setting
 allTime=true). Add asserts that timestamps are not < 0 cos it messes
 us up if they are (we already were checking for < 0 on construction but
 assert passed in timestamps are not < 0).

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
 Add constructor override that takes a TimeRangeTracker (set when flushing
 but not when compacting)

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 Add override creating an HFile in tmp that takes a TimeRangeTracker

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
 Add override for HFile Writer that takes a TimeRangeTracker Take it on
 construction instead of having it passed by a setter later (flags and
 reference set by the setter were not volatile... could have been prob
 in concurrent case)

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
 Log WARN if bad initial TimeRange value (and then 'fix' it)

M 
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
 A few tests to prove serialization works as expected and that we'll get a bad 
min if not constructed properly.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
 Handle OLDEST_TIMESTAMP explictly. Don't expect TimeRange to do it.

M 
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
 Refactor from junit3 to junit4 and add test for this weird case.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/496fd983
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/496fd983
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/496fd983

Branch: refs/heads/HBASE-14850
Commit: 496fd9837a0fb199a516758a632fecfe59b0b480
Parents: 3c39cbd
Author: stack 
Authored: Fri Jul 8 18:18:01 2016 -0700
Committer: stack 
Committed: Fri Jul 8 18:20:31 2016 -0700

--
 .../org/apache/hadoop/hbase/client/Query.java   |  4 +-
 .../org/apache/hadoop/hbase/HConstants.java |  5 +
 .../org/apache/hadoop/hbase/KeyValueUtil.java   |  2 +-
 .../org/apache/hadoop/hbase/io/TimeRange.java   | 99 
 .../hbase/mob/DefaultMobStoreFlusher.java   |  5 +-
 .../hbase/regionserver/DefaultStoreFlusher.java |  4 +-
 .../regionserver/ExplicitColumnTracker.java |  4 +-
 .../hadoop/hbase/regionserver/HStore.java   | 27 +-
 .../hbase/regionserver/ScanQueryMatcher.java| 11 ++-
 .../apache/hadoop/hbase/regionserver/Store.java | 20 +++-
 .../hadoop/hbase/regionserver/StoreFile.java|  1 -
 .../hbase/regionserver/StoreFileReader.java |  2 +-
 .../hbase/regionserver/StoreFileWriter.java | 61 
 .../hadoop/hbase/regionserver/StoreScanner.java |  3 +-
 .../hbase/regionserver/StripeStoreFlusher.java  |  4 +-
 .../hbase/regionserver/TimeRangeTracker.java| 23 ++---
 .../hbase/regionserver/TestQueryMatcher.java| 35 +--
 .../regionserver/TestTimeRangeTracker.java  | 40 +++-
 18 files changed, 250 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
index 53062a0..53dd2c1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
@@ -203,6 +203,4 @@ public abstract class Query extends OperationWithAttributes 
{
   public Map getColumnFamilyTimeRange() {
 return this.colFamTimeRangeMap;
   }
-
-
-}
+}
\ No newline at end of file

[34/50] [abbrv] hbase git commit: HBASE-15792 Add on a test for locating a region

[eclark]

HBASE-15792 Add on a test for locating a region

Summary:
Add on a TestUtil class that will start a cluster.
Add on the ability to run a shell command in a cluster.
Throw an exception if the table isn't the correct table when looking up a 
region.
Add a test to test that getting a region throws when the table doesn't exist.
Add a test to make sure that getting a region location works when the table 
does exist.

Test Plan:
Added unit tests.
Changed unit tests.

Differential Revision: https://reviews.facebook.net/D57891


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eb4cde49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eb4cde49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eb4cde49

Branch: refs/heads/HBASE-14850
Commit: eb4cde498f0fed8fd1e7ff35b71c69cabe09768d
Parents: d3a6efd
Author: Elliott Clark 
Authored: Mon May 9 12:06:54 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/start-local-hbase.sh|  5 +-
 .../connection/client-handler.cc|  3 +-
 hbase-native-client/core/BUCK   |  4 +-
 hbase-native-client/core/client.cc  |  1 +
 hbase-native-client/core/location-cache-test.cc | 29 ++-
 hbase-native-client/core/location-cache.cc  | 10 +++-
 hbase-native-client/core/location-cache.h   | 10 +++-
 hbase-native-client/core/test-env.cc| 45 
 hbase-native-client/test-util/BUCK  | 29 +++
 hbase-native-client/test-util/test-util.cc  | 54 
 hbase-native-client/test-util/test-util.h   | 51 ++
 11 files changed, 187 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/bin/start-local-hbase.sh
--
diff --git a/hbase-native-client/bin/start-local-hbase.sh 
b/hbase-native-client/bin/start-local-hbase.sh
index cfc71f9..49637d6 100755
--- a/hbase-native-client/bin/start-local-hbase.sh
+++ b/hbase-native-client/bin/start-local-hbase.sh
@@ -21,7 +21,8 @@
 rm -rf /tmp/hbase-*
 
 # Start the master/regionservers.
-$PWD/../bin/start-hbase.sh
+T_DIR=${1:-"/tmp/hbase-testing"}
+$PWD/../bin/start-hbase.sh -Dhbase.tmp.dir="${T_DIR}"
 
 until [ $(curl -s -o /dev/null -I -w "%{http_code}" 
http://localhost:16010/jmx) == "200" ]
 do
@@ -30,4 +31,4 @@ do
 done
 
 # This sucks, but master can easily be up and meta not be assigned yet.
-sleep 30
+sleep 10

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 2e3fcd3..3409b80 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,8 @@ using hbase::pb::GetResponse;
 using google::protobuf::Message;
 
 ClientHandler::ClientHandler(std::string user_name)
-: user_name_(user_name), serde_(), 
once_flag_(std::make_unique()),
+: user_name_(user_name), serde_(),
+  once_flag_(std::make_unique()),
   resp_msgs_(
   make_unique>(5000)) {}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 485f9ba..1c926e3 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -46,10 +46,10 @@ cxx_library(
 ], )
 cxx_test(name="location-cache-test",
  srcs=[
- "test-env.cc",
  "location-cache-test.cc",
  ],
- deps=[":core", ],
+ deps=[":core",
+   "//test-util:test-util", ],
  run_test_separately=True, )
 cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/client.cc
--
diff --git a/hbase-native-client/core/client.cc 
b/hbase-native-client/core/client.cc
index 1e80998..82ecd22 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -22,6 +22,7 @@
 #include 
 
 #include 
+#include 
 
 using namespace folly;
 using namespace std;

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/location-cache-test.cc
--
diff --git 

[46/50] [abbrv] hbase git commit: HBASE-15771 Document all the public classes

[eclark]

ad safe and does connection pooling. Current recommendations are 
to have only one Client per cluster around.
+ */
 class Client {
 public:
+
+  /**
+   * Create a new client.
+   * @param quorum_spec Where to connect to get Zookeeper bootstrap 
information.
+   */
   explicit Client(std::string quorum_spec);
 
 private:

http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/connection.cc
--
diff --git a/hbase-native-client/core/connection.cc 
b/hbase-native-client/core/connection.cc
deleted file mode 100644
index e9a28eb..000
--- a/hbase-native-client/core/connection.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "core/connection.h"
-
-void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }

http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/connection.h
--
diff --git a/hbase-native-client/core/connection.h 
b/hbase-native-client/core/connection.h
deleted file mode 100644
index 5c9d6b8..000
--- a/hbase-native-client/core/connection.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-class Connection {
-  char *zk_quorum;
-
-public:
-  void set_zk_quorum(char *zk_q);
-};

http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index e2a6251..583d305 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -128,15 +128,24 @@ LocationCache::LocateFromMeta(const TableName , const 
string ) {
   });
 }
 
+/**
+ * Filter to remove a service from the location cache and the connection cache 
on errors
+ * or on cloase.
+ */
 class RemoveServiceFilter
 : public ServiceFilter<std::unique_ptr, Response> {
 
 public:
+
+  /** Create a new filter. */
   RemoveServiceFilter(std::shared_ptr service, ServerName sn,
   ConnectionPool )
   : ServiceFilter<unique_ptr, Response>(service), sn_(sn),
 cp_(cp) {}
 
+  /**
+   * Close will remove the connection from all caches.
+   */
   folly::Future close() override {
 if (!released.exchange(true)) {
   return this->service_->close().then([this]() {
@@ -148,10 +157,13 @@ public:
 }
   }
 
+
+  /** Has this been closed */
   virtual bool isAvailable() override {
 return !released && service_->isAvailable();
   }
 
+  /** Send the message. */
   folly::Future operator()(unique_ptr req) override {
 // TODO(eclark): add in an on error handler that will
 // remove the region location from the cache if needed.
@@ -168,7 +180,7 @@ private:
 
 std::shared_ptr
 LocationCache::CreateLocation(const Response ) {
-  auto resp_msg = static_pointer_cast(resp.response());
+  auto resp_msg = static_pointer_cast(resp.resp_msg());
   auto  = resp_msg->results().Get(0);
   auto  = results.cell();
   auto ri = folly::to(cells.Get(0).value());

http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/location-cache.h
--

[22/50] [abbrv] hbase git commit: HBASE-14854 Read meta location from zk

[eclark]

HBASE-14854 Read meta location from zk


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/26ba4ed4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/26ba4ed4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/26ba4ed4

Branch: refs/heads/HBASE-14850
Commit: 26ba4ed461600a0bc8581d2ba9ba4230b71e992b
Parents: 27ee8ba
Author: Elliott Clark 
Authored: Sat Mar 5 00:09:08 2016 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile  |  20 ++-
 hbase-native-client/bin/start-docker.sh |   5 +-
 hbase-native-client/core/BUCK   | 106 +++---
 .../core/HBaseNativeClientTestEnv.cc|  42 --
 .../core/SampleNativeClientTest.cc  |  28 
 hbase-native-client/core/location-cache-test.cc |  14 ++
 hbase-native-client/core/location-cache.cc  |  67 +
 hbase-native-client/core/location-cache.h   |  35 +
 .../core/native-client-test-env.cc  |  42 ++
 .../core/simple-native-client-test.cc   |  28 
 hbase-native-client/core/test_env.h |   2 +
 hbase-native-client/if/BUCK |   1 +
 hbase-native-client/third-party/BUCK| 138 ++-
 13 files changed, 339 insertions(+), 189 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1364d22..36959a5 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM cpp_update 
+FROM pjameson/buck-folly-watchman
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5
@@ -25,20 +25,26 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g 
-fno-omit-frame-pointer -O3 -p
 RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
 RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git 
/usr/src/protobuf && \
   cd /usr/src/protobuf/ && \
+  ldconfig && \
   ./autogen.sh && \
-  ./configure --disable-shared && \
+  ./configure && \
   make && \
-  make check && \
-  make install
+  make install && \ 
+  make clean && \
+  rm -rf .git
+
 RUN cd /usr/src && \
   wget 
http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz 
&& \ 
   tar zxf zookeeper-3.4.8.tar.gz && \ 
   rm -rf zookeeper-3.4.8.tar.gz && \
   cd zookeeper-3.4.8 && \
   cd src/c && \
-  ./configure --disable-shared && \
+  ldconfig && \
+  ./configure && \
   make && \
   make install && \
-  make clean 
+  make clean
+
+RUN ldconfig
 
-WORKDIR /usr/local/src/hbase/hbase-native-client
+WORKDIR /usr/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 4426705..725ed6a 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -19,8 +19,11 @@
 set -e
 set -x
 
+# Try out some standard docker machine names that could work
 eval "$(docker-machine env docker-vm)"
 eval "$(docker-machine env dinghy)"
+
+# Build the image
 docker build -t hbase_native .
 
 
@@ -36,6 +39,6 @@ fi;
 
 docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-   -v ${PWD}/..:/usr/local/src/hbase \
+   -v ${PWD}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash

http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index ef027a1..817b5a0 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -15,52 +15,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+cxx_library(name="core",
+headers=[
+"admin.h",
+"client.h",
+"connection.h",
+"connection_attr.h",
+"delete.h",
+"get.h",
+"hbase_macros.h",
+"mutation.h",
+"put.h",
+"scanner.h",
+"location-cache.h",
+],
+srcs=[
+"admin.cc",
+"client.cc",
+

[20/50] [abbrv] hbase git commit: HBASE-14853 Add on protobuf to c++ chain

[eclark]

HBASE-14853 Add on protobuf to c++ chain


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e00e3a04
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e00e3a04
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e00e3a04

Branch: refs/heads/HBASE-14850
Commit: e00e3a046dddf46cef62d1e5ff3c2dda10b9e559
Parents: 543
Author: Elliott Clark 
Authored: Mon Dec 28 15:33:52 2015 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/start-docker.sh |   1 +
 hbase-native-client/core/BUCK   |   1 +
 hbase-native-client/core/client.cc  |   4 +
 hbase-native-client/core/client.h   |   2 +
 hbase-native-client/if/AccessControl.proto  | 123 +++
 hbase-native-client/if/Admin.proto  | 309 
 hbase-native-client/if/Aggregate.proto  |  63 ++
 hbase-native-client/if/Authentication.proto |  82 ++
 hbase-native-client/if/BUCK |  36 +
 hbase-native-client/if/Cell.proto   |  67 ++
 hbase-native-client/if/Client.proto | 472 +++
 hbase-native-client/if/ClusterId.proto  |  34 +
 hbase-native-client/if/ClusterStatus.proto  | 224 ++
 hbase-native-client/if/Comparator.proto |  74 ++
 hbase-native-client/if/Encryption.proto |  33 +
 hbase-native-client/if/ErrorHandling.proto  |  58 ++
 hbase-native-client/if/FS.proto |  45 ++
 hbase-native-client/if/Filter.proto | 170 
 hbase-native-client/if/HBase.proto  | 258 ++
 hbase-native-client/if/HFile.proto  |  49 ++
 hbase-native-client/if/LoadBalancer.proto   |  29 +
 hbase-native-client/if/MapReduce.proto  |  37 +
 hbase-native-client/if/Master.proto | 778 +++
 hbase-native-client/if/MasterProcedure.proto| 245 ++
 hbase-native-client/if/MultiRowMutation.proto   |  45 ++
 hbase-native-client/if/Procedure.proto  | 119 +++
 hbase-native-client/if/Quota.proto  |  76 ++
 hbase-native-client/if/RPC.proto| 136 
 hbase-native-client/if/RegionNormalizer.proto   |  28 +
 hbase-native-client/if/RegionServerStatus.proto | 158 
 hbase-native-client/if/RowProcessor.proto   |  45 ++
 hbase-native-client/if/SecureBulkLoad.proto |  72 ++
 hbase-native-client/if/Snapshot.proto   |  66 ++
 hbase-native-client/if/Tracing.proto|  33 +
 hbase-native-client/if/VisibilityLabels.proto   |  83 ++
 hbase-native-client/if/WAL.proto| 172 
 hbase-native-client/if/ZooKeeper.proto  | 155 
 hbase-native-client/rpc/CMakeLists.txt  |  17 -
 hbase-native-client/third-party/BUCK|   4 +-
 39 files changed, 4385 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index bf38912..4426705 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -20,6 +20,7 @@ set -e
 set -x
 
 eval "$(docker-machine env docker-vm)"
+eval "$(docker-machine env dinghy)"
 docker build -t hbase_native .
 
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 2e4e755..ef027a1 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -41,6 +41,7 @@ cxx_binary(
   "scanner.cc",
],
deps = [
+   "//if:if",
"//third-party:folly",
"//third-party:wangle",
],

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/client.cc
--
diff --git a/hbase-native-client/core/client.cc 
b/hbase-native-client/core/client.cc
index 98cf38a..a04daee 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -24,9 +24,13 @@
 #include 
 #include 
 
+#include "if/ZooKeeper.pb.h"
+
 using namespace folly;
+using namespace hbase::pb;
 
 int main(int argc, char *argv[]) {
+  MetaRegionServer mrs;
   google::ParseCommandLineFlags(, , true);
   google::InitGoogleLogging(argv[0]);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/client.h
--
diff --git 

[49/50] [abbrv] hbase git commit: HBASE-15724 Use explicit docker image

[eclark]

HBASE-15724 Use explicit docker image

Summary:
Set the tag for cpp docker image
Don't set the env on command line

Test Plan: Build it.

Differential Revision: https://reviews.facebook.net/D57315


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1069eb1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1069eb1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1069eb1

Branch: refs/heads/HBASE-14850
Commit: a1069eb13978213027d34a09e4a15f3fdea83f5d
Parents: 2aaadab
Author: Elliott Clark 
Authored: Wed Apr 27 10:51:05 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile  | 4 +++-
 hbase-native-client/bin/start-docker.sh | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a1069eb1/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1524c56..8e33e74 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,13 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman
+FROM pjameson/buck-folly-watchman:20160425
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5
 ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 
-pthread"
 ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 
-pthread"
 
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+
 RUN apt-get install -y vim maven inetutils-ping python-pip && \
   pip install yapf && \
   apt-get -qq clean && \

http://git-wip-us.apache.org/repos/asf/hbase/blob/a1069eb1/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 9cbd8b7..5c03eb9 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -53,7 +53,6 @@ docker build -t hbase_native .
 
 # After the image is built run the thing
 docker run -p 16010:16010/tcp \
-   -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash

[21/50] [abbrv] hbase git commit: HBASE-15401 Add Zookeeper to third party

[eclark]

HBASE-15401 Add Zookeeper to third party


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/27ee8baa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/27ee8baa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/27ee8baa

Branch: refs/heads/HBASE-14850
Commit: 27ee8baaafecfaf4709ecefd1ef2099a82ca3d39
Parents: e00e3a0
Author: Elliott Clark 
Authored: Mon Mar 7 14:58:21 2016 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile   | 26 -
 hbase-native-client/third-party/BUCK | 96 ++-
 2 files changed, 68 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/27ee8baa/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 5f17f04..1364d22 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,8 +15,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman
+FROM cpp_update 
 
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim 
maven inetutils-ping
+ARG CC=/usr/bin/gcc-5
+ARG CXX=/usr/bin/g++-5
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
+
+RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
+RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git 
/usr/src/protobuf && \
+  cd /usr/src/protobuf/ && \
+  ./autogen.sh && \
+  ./configure --disable-shared && \
+  make && \
+  make check && \
+  make install
+RUN cd /usr/src && \
+  wget 
http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz 
&& \ 
+  tar zxf zookeeper-3.4.8.tar.gz && \ 
+  rm -rf zookeeper-3.4.8.tar.gz && \
+  cd zookeeper-3.4.8 && \
+  cd src/c && \
+  ./configure --disable-shared && \
+  make && \
+  make install && \
+  make clean 
 
 WORKDIR /usr/local/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/27ee8baa/hbase-native-client/third-party/BUCK
--
diff --git a/hbase-native-client/third-party/BUCK 
b/hbase-native-client/third-party/BUCK
index b7baa86..e577a5f 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -15,68 +15,60 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-def add_system_libs(names = []):
+def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = 
[], exported_linker_flags = []):
 rules = []
 for name in names:
-prebuilt_cxx_library(
-name = name,
-lib_name = name,
-lib_dir = "/usr/lib/x86_64-linux-gnu",
-visibility = [ 'PUBLIC', ],
-)
-rules.append(":" + name)
-
+gen_rule_name = "gen_lib{}".format(name)
+genrule(
+  name = gen_rule_name,
+  out = gen_rule_name,
+  bash = "mkdir -p $OUT && cp {}/lib{}.* 
$OUT".format(lib_dir,name),
+)
+prebuilt_cxx_library(
+  name = name,
+  lib_name = name,
+  lib_dir = '$(location :{})'.format(gen_rule_name),
+  force_static = True,
+  deps = deps,
+  visibility = [ 'PUBLIC' ],
+  exported_linker_flags = exported_linker_flags,
+)
+rules.append(":" + name)
 return rules
 
 system_libs = [
+"unwind",
+"lzma",
+]
+local_libs = [
 "double-conversion",
 "glog",
-"protobuf",
 "gflags",
-"unwind",
-"lzma",
+"protobuf",
+"zookeeper_mt",
 "boost_regex",
 ]
-tp_dep_rules = add_system_libs(system_libs)
-prebuilt_cxx_library(
-name = "folly",
-lib_name = "folly",
-lib_dir = "/usr/local/lib",
-deps = tp_dep_rules,
-exported_linker_flags = [
-"-pthread",
-"-lstdc++",
-],
-visibility = [
-'PUBLIC',
-]
-)
-prebuilt_cxx_library(
-name = "follybenchmark",
-lib_name = "follybenchmark",
-lib_dir = "/usr/local/lib",
-deps = tp_dep_rules + [":folly"],
-exported_linker_flags = [
-"-pthread",
-"-lstdc++",
-],
-

[36/50] [abbrv] hbase git commit: HBASE-15822 Move to the latest docker base image

[eclark]

HBASE-15822 Move to the latest docker base image


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bb7fa8b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bb7fa8b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bb7fa8b

Branch: refs/heads/HBASE-14850
Commit: 5bb7fa8b832a3134f83b7d94404494ba07f0bd4d
Parents: df14a6a
Author: Elliott Clark 
Authored: Tue May 31 10:42:45 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile   | 2 +-
 hbase-native-client/if/RPC.proto | 1 +
 hbase-native-client/if/WAL.proto | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index be0cbc6..ca87523 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman:20160425
+FROM pjameson/buck-folly-watchman:20160511
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5

http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/if/RPC.proto
--
diff --git a/hbase-native-client/if/RPC.proto b/hbase-native-client/if/RPC.proto
index 59bb03d..8413d25 100644
--- a/hbase-native-client/if/RPC.proto
+++ b/hbase-native-client/if/RPC.proto
@@ -125,6 +125,7 @@ message RequestHeader {
   // 0 is NORMAL priority.  200 is HIGH.  If no priority, treat it as NORMAL.
   // See HConstants.
   optional uint32 priority = 6;
+  optional uint32 timeout = 7;
 }
 
 message ResponseHeader {

http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/if/WAL.proto
--
diff --git a/hbase-native-client/if/WAL.proto b/hbase-native-client/if/WAL.proto
index cb9bd8f..2061b22 100644
--- a/hbase-native-client/if/WAL.proto
+++ b/hbase-native-client/if/WAL.proto
@@ -44,9 +44,9 @@ message WALKey {
   required uint64 log_sequence_number = 3;
   required uint64 write_time = 4;
   /*
-  This parameter is deprecated in favor of clusters which 
+  This parameter is deprecated in favor of clusters which
   contains the list of clusters that have consumed the change.
-  It is retained so that the log created by earlier releases (0.94) 
+  It is retained so that the log created by earlier releases (0.94)
   can be read by the newer releases.
   */
   optional UUID cluster_id = 5 [deprecated=true];
@@ -132,6 +132,7 @@ message StoreDescriptor {
   required bytes family_name = 1;
   required string store_home_dir = 2; //relative to region dir
   repeated string store_file = 3; // relative to store dir
+  optional uint64 store_file_size_bytes = 4; // size of store file
 }
 
 /**

[01/50] [abbrv] hbase git commit: HBASE-16171 Fix the potential problems in TestHCM.testConnectionCloseAllowsInterrupt (Colin Ma) [Forced Update!]

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 114817253 -> 0eb150f9d (forced update)


HBASE-16171 Fix the potential problems in 
TestHCM.testConnectionCloseAllowsInterrupt (Colin Ma)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e0d97e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e0d97e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e0d97e0

Branch: refs/heads/HBASE-14850
Commit: 5e0d97e0e95434b03ba886279180a031ed997dfd
Parents: a33097e
Author: stack 
Authored: Fri Jul 8 10:17:55 2016 -0700
Committer: stack 
Committed: Fri Jul 8 10:18:36 2016 -0700

--
 .../src/test/java/org/apache/hadoop/hbase/client/TestHCM.java | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5e0d97e0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 4723fa8..1b20b76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -464,6 +464,10 @@ public class TestHCM {
 c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 1); // don't wait between retries.
 c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0); // Server do not really 
expire
 c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
+// to avoid the client to be stuck when do the Get
+c2.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 1);
+c2.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 1);
+c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000);
 
 Connection connection = ConnectionFactory.createConnection(c2);
 final Table table = connection.getTable(tableName);
@@ -488,6 +492,9 @@ public class TestHCM {
 done++;
 if (done % 100 == 0)
   LOG.info("done=" + done);
+// without the sleep, will cause the exception for too many files 
in
+// org.apache.hadoop.hdfs.server.datanode.DataXceiver
+Thread.sleep(100);
   }
 } catch (Throwable t) {
   failed.set(t);

[30/50] [abbrv] hbase git commit: HBASE-15696 Move region location cache serialization into serde

[eclark]

f/RPC.pb.h"
 #include "if/Client.pb.h"
+#include "if/RPC.pb.h"
 
 using namespace hbase;
 using namespace folly;
@@ -60,6 +60,9 @@ void ClientHandler::read(Context *ctx, std::unique_ptr 
buf) {
   }
 }
 
+
+// TODO(eclark): Figure out how to handle the
+// network errors that are going to come.
 Future ClientHandler::write(Context *ctx, Request r) {
   // Keep track of if we have sent the header.
   if (UNLIKELY(need_send_header_)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/client-handler.h
--
diff --git a/hbase-native-client/connection/client-handler.h 
b/hbase-native-client/connection/client-handler.h
index 38c5725..dbaf5a0 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -22,8 +22,8 @@
 
 #include 
 
-#include "serde/client-serializer.h"
 #include "serde/client-deserializer.h"
+#include "serde/client-serializer.h"
 
 // Forward decs.
 namespace hbase {

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/pipeline.h
--
diff --git a/hbase-native-client/connection/pipeline.h 
b/hbase-native-client/connection/pipeline.h
index 68ade48..8114fab 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -18,8 +18,8 @@
  */
 #pragma once
 
-#include 
 #include 
+#include 
 
 #include "connection/request.h"
 #include "connection/response.h"

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/client.cc
--
diff --git a/hbase-native-client/core/client.cc 
b/hbase-native-client/core/client.cc
index 893894f..266c239 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -21,8 +21,8 @@
 
 #include 
 #include 
-#include 
 #include 
+#include 
 #include 
 
 #include 
@@ -36,5 +36,5 @@ using namespace hbase::pb;
 namespace hbase {
 
 Client::Client(string quorum_spec)
-: location_cache(quorum_spec, wangle::getCPUExecutor()) {}
+: location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/client.h
--
diff --git a/hbase-native-client/core/client.h 
b/hbase-native-client/core/client.h
index c2dc226..b583285 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -19,8 +19,8 @@
 
 #pragma once
 
-#include 
 #include 
+#include 
 
 #include 
 
@@ -36,7 +36,7 @@ public:
   folly::Future get(const GetRequest _request);
 
 private:
-  LocationCache location_cache;
+  LocationCache location_cache_;
 };
 
 } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/location-cache-test.cc
--
diff --git a/hbase-native-client/core/location-cache-test.cc 
b/hbase-native-client/core/location-cache-test.cc
index 70ca6f1..f3166fb 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -16,8 +16,8 @@
  * limitations under the License.
  *
  */
-#include 
 #include 
+#include 
 #include 
 
 #include "location-cache.h"

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index 52e86e3..5925f4a 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -16,11 +16,13 @@
  * limitations under the License.
  *
  */
-#include "location-cache.h"
+#include "core/location-cache.h"
 
 #include 
+#include 
 
 #include "if/ZooKeeper.pb.h"
+#include "serde/zk-deserializer.h"
 
 using namespace std;
 using namespace folly;
@@ -28,7 +30,7 @@ using namespace hbase::pb;
 using namespace hbase;
 
 // TODO(eclark): make this configurable on client creation
-static const char META_LOCATION[] = "/hbase/meta-region-server";
+static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
 
 LocationCache::LocationCache(string quorum_spec,
  shared_ptr executor)
@@ -66,45 +68,25 @@ void LocationCache::RefreshMetaLocation() {
 }
 
 ServerName LocationCache::ReadMetaLocation() {
-  char contents[4096];
+  auto buf = IOBuf::create(4096);
+  ZkDeserializer derser;
+
   // This needs to be int rather than size_t as that's what ZK expects.
-  int len = sizeof(contents);
+  int len = buf->

[50/50] [abbrv] hbase git commit: HBASE-16174 Hook cell test up, and fix broken cell test.

[eclark]

HBASE-16174 Hook cell test up, and fix broken cell test.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0eb150f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0eb150f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0eb150f9

Branch: refs/heads/HBASE-14850
Commit: 0eb150f9dce5c0dc3dae67a9da7b28f912853952
Parents: fa3ab42
Author: Elliott Clark 
Authored: Tue Jul 5 11:13:01 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/core/BUCK | 8 
 hbase-native-client/core/cell-test.cc | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0eb150f9/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 1c926e3..c615426 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -20,6 +20,7 @@ cxx_library(
 name="core",
 exported_headers=[
 "client.h",
+"cell.h",
 "hbase_macros.h",
 "region-location.h",
 "location-cache.h",
@@ -28,6 +29,7 @@ cxx_library(
 "meta-utils.h",
 ],
 srcs=[
+"cell.cc",
 "client.cc",
 "location-cache.cc",
 "meta-utils.cc",
@@ -51,6 +53,12 @@ cxx_test(name="location-cache-test",
  deps=[":core",
"//test-util:test-util", ],
  run_test_separately=True, )
+cxx_test(name="cell-test",
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
 cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/0eb150f9/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
index cbe50eb..49f567b 100644
--- a/hbase-native-client/core/cell-test.cc
+++ b/hbase-native-client/core/cell-test.cc
@@ -100,7 +100,7 @@ TEST(CellTest, MultipleCellsTest) {
 
 TEST(CellTest, CellRowTest) {
   std::string row = "only-row";
-  std::string family = "";
+  std::string family = "D";
   std::string column = "";
   std::string value = "";
   long timestamp = std::numeric_limits::max();

[32/50] [abbrv] hbase git commit: HBASE-15774 Fix Upgrade lock usage in connection pool.

[eclark]

e sure there are not stale connections hanging around.
+connections_.erase(sn);
+
+// Nope we are the ones who should create the new connection.
 auto client = cf_->MakeBootstrap();
 auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
 clients_.insert(std::make_pair(sn, client));
 connections_.insert(std::make_pair(sn, dispatcher));
 return dispatcher;
   }
-  return found->second;
 }
 
-void ConnectionPool::close(const ServerName ) {
+void ConnectionPool::Close(const ServerName ) {
   SharedMutexWritePriority::WriteHolder holder{map_mutex_};
 
   auto found = connections_.find(sn);

http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/connection/connection-pool.h
--
diff --git a/hbase-native-client/connection/connection-pool.h 
b/hbase-native-client/connection/connection-pool.h
index 60f00de..605a81b 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -80,14 +80,16 @@ public:
* Get a connection to the server name. Start time is ignored.
* This can be a blocking operation for a short time.
*/
-  std::shared_ptr get(const hbase::pb::ServerName );
+  std::shared_ptr Get(const hbase::pb::ServerName );
 
   /**
* Close/remove a connection.
*/
-  void close(const hbase::pb::ServerName );
+  void Close(const hbase::pb::ServerName );
 
 private:
+  std::shared_ptr GetCached(const hbase::pb::ServerName& sn);
+  std::shared_ptr GetNew(const hbase::pb::ServerName& sn);
   std::unordered_map<hbase::pb::ServerName, std::shared_ptr,
  ServerNameHash, ServerNameEquals>
   connections_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/core/client.h
--
diff --git a/hbase-native-client/core/client.h 
b/hbase-native-client/core/client.h
index 4db82c4..ba24bb9 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -48,9 +48,9 @@ public:
   ~Client();
 
 private:
-  LocationCache location_cache_;
   std::shared_ptr cpu_executor_;
   std::shared_ptr io_executor_;
+  LocationCache location_cache_;
 };
 
 } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index 9f2a0ef..6ba8add 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -115,7 +115,7 @@ Future<std::shared_ptr>
 LocationCache::LocateFromMeta(const TableName , const string ) {
   return this->LocateMeta()
   .via(cpu_executor_.get())
-  .then([this](ServerName sn) { return this->cp_.get(sn); })
+  .then([this](ServerName sn) { return this->cp_.Get(sn); })
 .then([tn, row, this](std::shared_ptr service) {
 return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
   })
@@ -134,67 +134,25 @@ LocationCache::LocateFromMeta(const TableName , const 
string ) {
   })
   .then([this](std::shared_ptr rl) {
 // Now fill out the connection.
-rl->set_service(cp_.get(rl->server_name()));
+rl->set_service(cp_.Get(rl->server_name()));
 return rl;
   });
 }
 
-/**
- * Filter to remove a service from the location cache and the connection cache
- * on errors
- * or on cloase.
- */
-class RemoveServiceFilter
-: public ServiceFilter<std::unique_ptr, Response> {
-
-public:
-  /** Create a new filter. */
-  RemoveServiceFilter(std::shared_ptr service, ServerName sn,
-  ConnectionPool )
-  : ServiceFilter<unique_ptr, Response>(service), sn_(sn),
-cp_(cp) {}
-
-  /**
-   * Close will remove the connection from all caches.
-   */
-  folly::Future close() override {
-if (!released.exchange(true)) {
-  return this->service_->close().then([this]() {
-// TODO(eclark): remove the service from the meta cache.
-this->cp_.close(this->sn_);
-  });
-} else {
-  return folly::makeFuture();
-}
-  }
-
-  /** Has this been closed */
-  virtual bool isAvailable() override {
-return !released && service_->isAvailable();
-  }
-
-  /** Send the message. */
-  folly::Future operator()(unique_ptr req) override {
-// TODO(eclark): add in an on error handler that will
-// remove the region location from the cache if needed.
-// Also close the connection if this is likely to be an error
-// that needs to get a new connection.
-return (*this->service_)(std::move(req));
-  }
-
-private:
-  std::atomic released{false};
-  hbase::pb::ServerName sn_;
-  ConnectionPool _;
-};
-
 std::shar

[38/50] [abbrv] hbase git commit: HBASE-15750 Add on meta deserialization

[eclark]

p;](std::shared_ptr service) {
-return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
+return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
   })
   .then([&](Response resp) {
 // take the protobuf response and make it into
 // a region location.
-return this->parse_response(std::move(resp));
+return this->CreateLocation(std::move(resp));
   });
 }
 
-RegionLocation LocationCache::parse_response(const Response ) {
+class RemoveServiceFilter
+: public ServiceFilter<std::unique_ptr, Response> {
+
+public:
+  RemoveServiceFilter(std::shared_ptr service, ServerName sn,
+  ConnectionPool )
+  : ServiceFilter<unique_ptr, Response>(service), sn_(sn),
+cp_(cp) {}
+
+  folly::Future close() override {
+if (!released.exchange(true)) {
+  return this->service_->close().then([this]() {
+// TODO(eclark): remove the service from the meta cache.
+this->cp_.close(this->sn_);
+  });
+} else {
+  return folly::makeFuture();
+}
+  }
+
+  virtual bool isAvailable() override {
+return !released && service_->isAvailable();
+  }
+
+  folly::Future operator()(unique_ptr req) override {
+// TODO(eclark): add in an on error handler that will
+// remove the region location from the cache if needed.
+// Also close the connection if this is likely to be an error
+// that needs to get a new connection.
+return (*this->service_)(std::move(req));
+  }
+
+private:
+  std::atomic released{false};
+  hbase::pb::ServerName sn_;
+  ConnectionPool _;
+};
+
+std::shared_ptr
+LocationCache::CreateLocation(const Response ){
   auto resp_msg = static_pointer_cast(resp.response());
+  auto  = resp_msg->results().Get(0);
+  auto  = results.cell();
   LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
-  return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
+  auto ri = folly::to(cells.Get(0).value());
+  auto sn = folly::to(cells.Get(1).value());
+
+  LOG(ERROR) << "RegionInfo = " << ri.DebugString();
+  LOG(ERROR) << "ServerName = " << sn.DebugString();
+  auto wrapped = make_shared(cp_.get(sn), sn, this->cp_);
+  return std::make_shared(std::move(ri), std::move(sn), 
wrapped);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/location-cache.h
--
diff --git a/hbase-native-client/core/location-cache.h 
b/hbase-native-client/core/location-cache.h
index cfd6838..99b5e5e 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -29,8 +29,8 @@
 
 #include "connection/connection-pool.h"
 #include "core/meta-utils.h"
-#include "core/table-name.h"
 #include "core/region-location.h"
+#include "serde/table-name.h"
 
 namespace hbase {
 
@@ -48,14 +48,14 @@ public:
   // Meta Related Methods.
   // These are only public until testing is complete
   folly::Future LocateMeta();
-  folly::Future locateFromMeta(const hbase::pb::TableName ,
+  folly::Future<std::shared_ptr> LocateFromMeta(const 
hbase::pb::TableName ,
const std::string );
-  RegionLocation parse_response(const Response );
   void InvalidateMeta();
 
 private:
   void RefreshMetaLocation();
   hbase::pb::ServerName ReadMetaLocation();
+  std::shared_ptr CreateLocation(const Response );
 
   std::string quorum_spec_;
   std::shared_ptr executor_;
@@ -64,7 +64,6 @@ private:
   ConnectionPool cp_;
   MetaUtil meta_util_;
 
-
   // TODO: migrate this to a smart pointer with a deleter.
   zhandle_t *zk_;
 };

http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/meta-utils.cc
--
diff --git a/hbase-native-client/core/meta-utils.cc 
b/hbase-native-client/core/meta-utils.cc
index d2fdd88..1325d83 100644
--- a/hbase-native-client/core/meta-utils.cc
+++ b/hbase-native-client/core/meta-utils.cc
@@ -23,25 +23,26 @@
 
 #include "connection/request.h"
 #include "connection/response.h"
-#include "core/table-name.h"
 #include "if/Client.pb.h"
+#include "serde/table-name.h"
 
 using hbase::pb::TableName;
 using hbase::MetaUtil;
 using hbase::Request;
 using hbase::Response;
 using hbase::pb::ScanRequest;
+using hbase::pb::ServerName;
 using hbase::pb::RegionSpecifier_RegionSpecifierType;
 
 static const std::string META_REGION = "1588230740";
 
-std::string MetaUtil::region_lookup_rowkey(const TableName ,
+std::string MetaUtil::RegionLookupRowkey(const TableName ,
const std::string ) const {
   return folly::to(tn, ",", row, ",", &q

[42/50] [abbrv] hbase git commit: HBASE-15926 Added ASF headers which were missed in last commit

[eclark]

HBASE-15926 Added ASF headers which were missed in last commit

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b6341d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b6341d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b6341d4

Branch: refs/heads/HBASE-14850
Commit: 2b6341d4db3f3cf045b99fb6f71d431098ce7bf1
Parents: 5bb7fa8
Author: Sudeep Sunthankar 
Authored: Thu Jun 2 20:22:37 2016 +1000
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Makefile| 17 +
 hbase-native-client/Makefile.protos | 17 +
 2 files changed, 34 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2b6341d4/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 7e68b6a..64cef06 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 #use "gcc" to compile source files
 CC:=g++
 LD:=g++

http://git-wip-us.apache.org/repos/asf/hbase/blob/2b6341d4/hbase-native-client/Makefile.protos
--
diff --git a/hbase-native-client/Makefile.protos 
b/hbase-native-client/Makefile.protos
index 4f1727a..2c6316b 100644
--- a/hbase-native-client/Makefile.protos
+++ b/hbase-native-client/Makefile.protos
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 #use "gcc" to compile source files
 CC:=g++
 LD:=g++

[13/50] [abbrv] hbase git commit: HBASE-15604 Add a good readme on the build.

[eclark]

HBASE-15604 Add a good readme on the build.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bfabc715
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bfabc715
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bfabc715

Branch: refs/heads/HBASE-14850
Commit: bfabc715c30350ec1ee1953826ab844ceb60d750
Parents: 45cf565
Author: Elliott Clark 
Authored: Fri Apr 8 13:44:45 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/BUILDING.md | 63 
 1 file changed, 63 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bfabc715/hbase-native-client/BUILDING.md
--
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
new file mode 100644
index 000..20ef2a0
--- /dev/null
+++ b/hbase-native-client/BUILDING.md
@@ -0,0 +1,63 @@
+
+
+#Building HBase native client
+
+The HBase native client build using buck and produces a linux library.
+
+
+# Dependencies
+
+The easiest way to build hbase-native-client is to
+use [Docker](https://www.docker.com/). This will mean that any platform
+with docker can be used to build the hbase-native-client. It will also
+mean that you're building with the same versions of all dependencies that
+the client is tested with.
+
+On OSX the current boot2docker solution will work however it's pretty
+slow. If things get too slow using 
[dinghy](https://github.com/codekitchen/dinghy)
+can help speed things up by using nfs.
+
+If possible pairing virtual box with dinghy will result in the fastest,
+most stable docker environment. However none of it is needed.
+
+# Building using docker
+
+To start with make sure that you have built the java project using
+`mvn package -DskipTests`. That will allow all tests to spin up a standalone
+hbase instance from the jar's created.
+
+Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
+that will build the docker development environment and when complete will
+drop you into a shell on a linux vm with all the tools needed installed.
+
+# Buck
+
+From then on we can use [buck](https://buckbuild.com/) to build everything.
+For example:
+```
+buck build //core:core
+buck test --all
+buck build //core:simple-client
+```
+
+That will build the library, then build and test everything, then build
+the simple-client binary. Buck will find all modules used, and compile 
+them in parallel, caching the results. Output from buck is in the buck-out
+foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs

[14/50] [abbrv] hbase git commit: HBASE-15718 Add on TableName implementation and tests

[eclark]

HBASE-15718 Add on TableName implementation and tests

Summary:
Table name will be needed to look up a row in meta. So this
patch adds on the implementation for creating that lookup
row. It also adds tests for TableName

Test Plan: Added on a unit test

Differential Revision: https://reviews.facebook.net/D57285


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2aaadab1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2aaadab1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2aaadab1

Branch: refs/heads/HBASE-14850
Commit: 2aaadab1519cdb11dd2c8df6cc8e61c1ecf4e82d
Parents: 7cae46e
Author: Elliott Clark <ecl...@apache.org>
Authored: Tue Apr 26 14:51:06 2016 -0700
Committer: Elliott Clark <ecl...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-handler.cc|  1 -
 hbase-native-client/core/BUCK   | 69 +++-
 hbase-native-client/core/meta-utils.cc  | 31 +
 hbase-native-client/core/meta-utils.h   | 30 +
 hbase-native-client/core/simple-client.cc   |  4 +-
 hbase-native-client/core/table-name-test.cc | 50 ++
 hbase-native-client/core/table-name.cc  | 15 +
 hbase-native-client/core/table-name.h   | 25 ++-
 hbase-native-client/if/Master.proto | 13 
 hbase-native-client/if/ZooKeeper.proto  | 10 ++-
 10 files changed, 211 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 7b7794d..205993a7 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -60,7 +60,6 @@ void ClientHandler::read(Context *ctx, std::unique_ptr 
buf) {
   }
 }
 
-
 // TODO(eclark): Figure out how to handle the
 // network errors that are going to come.
 Future ClientHandler::write(Context *ctx, Request r) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 195fc5c..9db6fda 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -16,44 +16,49 @@
 # limitations under the License.
 
 # This is the main library.
-cxx_library(name="core",
-exported_headers=[
-"client.h",
-"connection.h",
-"get-request.h",
-"get-result.h",
-"hbase_macros.h",
-"location-cache.h",
-"table-name.h",
-],
-srcs=[
-"client.cc",
-"get-request.cc",
-"get-result.cc",
-"location-cache.cc",
-"table-name.cc",
-],
-deps=[
-"//connection:connection",
-"//if:if",
-"//serde:serde",
-"//third-party:folly",
-"//third-party:wangle",
-"//third-party:zookeeper_mt",
-],
-visibility=[
-'PUBLIC',
-], )
-
+cxx_library(
+name="core",
+exported_headers=[
+"client.h",
+"connection.h",
+"get-request.h",
+"get-result.h",
+"hbase_macros.h",
+"location-cache.h",
+"table-name.h",
+# TODO: move this out of exported
+# Once meta lookup works
+"meta-utils.h",
+],
+srcs=[
+"client.cc",
+"get-request.cc",
+"get-result.cc",
+"location-cache.cc",
+"meta-utils.cc",
+"table-name.cc",
+],
+deps=[
+"//connection:connection",
+"//if:if",
+"//serde:serde",
+"//third-party:folly",
+"//third-party:wangle",
+"//third-party:zookeeper_mt",
+],
+visibility=[
+'PUBLIC',
+], )
 cxx_test(name="location-cache-test",
  srcs=[
  "test-env.cc",
  "location-cache-test.cc",
  ],
- de

[16/50] [abbrv] hbase git commit: HBASE-14852 Update build env

[eclark]

http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_result.h
--
diff --git a/hbase-native-client/src/async/hbase_result.h 
b/hbase-native-client/src/async/hbase_result.h
deleted file mode 100644
index eecbbb3..000
--- a/hbase-native-client/src/async/hbase_result.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_RESULT_H_
-#define ASYNC_HBASE_RESULT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-HBASE_API int32_t hb_result_destroy(hb_result_t result);
-
-HBASE_API int32_t hb_result_get_cells(hb_result_t result,
-hb_cell_t ** cell_ptr, size_t * num_cells);
-
-HBASE_API int32_t hb_result_get_table(hb_result_t result,
-char ** table, size_t * table_length);
-HBASE_API int32_t hb_result_get_namespace(hb_result_t result,
-char ** name_space, size_t * name_space_length);
-
-#ifdef __cplusplus
-}  // extern "C"
-#endif  // __cplusplus
-
-#endif  // ASYNC_HBASE_RESULT_H_

http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_scanner.cc
--
diff --git a/hbase-native-client/src/async/hbase_scanner.cc 
b/hbase-native-client/src/async/hbase_scanner.cc
deleted file mode 100644
index 5a8e555..000
--- a/hbase-native-client/src/async/hbase_scanner.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "async/hbase_scanner.h"
-
-#include 
-
-#include "core/hbase_types.h"
-#include "core/scanner.h"
-
-int32_t hb_scanner_create(hb_scanner_t * scanner_ptr) {
-  (*scanner_ptr) = reinterpret_cast(new Scanner());
-  return (*scanner_ptr != NULL)?0:1;
-}
-
-HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
-char * table, size_t table_length) {
-  return 0;
-}
-
-HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
-char * name_space, size_t name_space_length) {
-  return 0;
-}
-
-int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
-unsigned char * start_row, size_t start_row_length) {
-  return 0;
-}
-
-int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
-unsigned char * end_row, size_t end_row_length) {
-  return 0;
-}
-
-int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
-size_t cache_size) {
-  return 0;
-}
-
-int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
-int8_t num_versions) {
-  return 0;
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_scanner.h
--
diff --git a/hbase-native-client/src/async/hbase_scanner.h 
b/hbase-native-client/src/async/hbase_scanner.h
deleted file mode 100644
index cd3f544..000
--- a/hbase-native-client/src/async/hbase_scanner.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * 

[06/50] [abbrv] hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells

[eclark]

HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9cf012cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9cf012cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9cf012cd

Branch: refs/heads/HBASE-14850
Commit: 9cf012cd0086e054a66f54cb54771af2381fade0
Parents: 6329697
Author: Yu Li 
Authored: Sun Jul 10 13:45:17 2016 +0800
Committer: Yu Li 
Committed: Sun Jul 10 13:45:17 2016 +0800

--
 .../hbase/regionserver/AbstractMemStore.java| 15 +++
 .../hbase/regionserver/HeapMemStoreLAB.java | 11 +
 .../hbase/regionserver/MemStoreCompactor.java   |  3 ++-
 .../hbase/regionserver/MutableSegment.java  |  6 +++--
 .../hadoop/hbase/regionserver/Segment.java  | 23 ++---
 .../hbase/regionserver/TestDefaultMemStore.java | 26 
 6 files changed, 73 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index d25f960..79e95af 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,7 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-return internalAdd(toAdd);
+boolean useMSLAB = (toAdd != cell);
+return internalAdd(toAdd, useMSLAB);
   }
 
   /**
@@ -156,7 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long delete(Cell deleteCell) {
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-long s = internalAdd(toAdd);
+boolean useMSLAB = (toAdd != deleteCell);
+long s = internalAdd(toAdd, useMSLAB);
 return s;
   }
 
@@ -243,7 +245,7 @@ public abstract class AbstractMemStore implements MemStore {
 // hitting OOME - see TestMemStore.testUpsertMSLAB for a
 // test that triggers the pathological case if we don't avoid MSLAB
 // here.
-long addedSize = internalAdd(cell);
+long addedSize = internalAdd(cell, false);
 
 // Get the Cells for the row/family/qualifier regardless of timestamp.
 // For this case we want to clean up any other puts
@@ -387,9 +389,12 @@ public abstract class AbstractMemStore implements MemStore 
{
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+   * @param toAdd the cell to add
+   * @param useMSLAB whether using MSLAB
+   * @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd) {
-long s = active.add(toAdd);
+  private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
+long s = active.add(toAdd, useMSLAB);
 setOldestEditTimeToNow();
 checkActiveSize();
 return s;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
index f22a6e5..625811a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.util.ByteRange;
 import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 /**
@@ -206,6 +207,11 @@ public class HeapMemStoreLAB implements MemStoreLAB {
 }
   }
 
+  @VisibleForTesting
+  Chunk getCurrentChunk() {
+return this.curChunk.get();
+  }
+
   /**
* A chunk of memory out of which allocations are sliced.
*/
@@ -311,5 +317,10 @@ public class HeapMemStoreLAB implements MemStoreLAB {
 " allocs=" + allocCount.get() + "waste=" +
 (data.length - nextFreeOffset.get());
 }
+
+@VisibleForTesting
+int getNextFreeOffset() {
+  return this.nextFreeOffset.get();
+}
   }
 }

[11/50] [abbrv] hbase git commit: HBASE-15823 Use call once for user util

[eclark]

HBASE-15823 Use call once for user util


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dea1eb64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dea1eb64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dea1eb64

Branch: refs/heads/HBASE-14850
Commit: dea1eb64bd6ef85ef0ca9a4c1f5c7b30043ba953
Parents: e718c7d
Author: Elliott Clark 
Authored: Fri May 13 13:07:03 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/utils/user-util.cc | 13 ++---
 hbase-native-client/utils/user-util.h  |  6 ++
 2 files changed, 4 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/dea1eb64/hbase-native-client/utils/user-util.cc
--
diff --git a/hbase-native-client/utils/user-util.cc 
b/hbase-native-client/utils/user-util.cc
index 3d963b3..9e170e0 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -27,22 +27,14 @@
 using namespace hbase;
 using namespace std;
 
-UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
+UserUtil::UserUtil() : once_flag_{}, user_name_{"drwho"} {}
 
 string UserUtil::user_name() {
-  if (!init_) {
-compute_user_name();
-  }
+  std::call_once(once_flag_, [this]() { compute_user_name(); });
   return user_name_;
 }
 
 void UserUtil::compute_user_name() {
-  lock_guard lock(m_);
-
-  if (init_) {
-return;
-  }
-
   // According to the man page of getpwuid
   // this should never be free'd
   //
@@ -52,6 +44,5 @@ void UserUtil::compute_user_name() {
   // make sure that we got something.
   if (passwd && passwd->pw_name) {
 user_name_ = string{passwd->pw_name};
-init_ = true;
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/dea1eb64/hbase-native-client/utils/user-util.h
--
diff --git a/hbase-native-client/utils/user-util.h 
b/hbase-native-client/utils/user-util.h
index 0b4cc73..fdfc0c8 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -19,8 +19,7 @@
 
 #pragma once
 
-#include 
-#include 
+#include 
 #include 
 
 namespace hbase {
@@ -49,8 +48,7 @@ private:
* Compute the username. This will block.
*/
   void compute_user_name();
-  std::atomic init_;
+  std::once_flag once_flag_;
   std::string user_name_;
-  std::mutex m_;
 };
 } // namespace hbase

[25/50] [abbrv] hbase git commit: HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.

[eclark]

HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.

Summary: On every start of docker make sure that protos are up to date.

Test Plan: Added the script and it updated the protos, but didn't delete BUCK.

Differential Revision: https://reviews.facebook.net/D56763


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bbeaacb6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bbeaacb6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bbeaacb6

Branch: refs/heads/HBASE-14850
Commit: bbeaacb6d43b29746abc78f31c422c7706938f8a
Parents: 098028c
Author: Elliott Clark <ecl...@apache.org>
Authored: Wed Apr 13 15:59:13 2016 -0700
Committer: Elliott Clark <ecl...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/copy-protobuf.sh |   8 ++
 hbase-native-client/bin/start-docker.sh  |  22 +++-
 hbase-native-client/if/Admin.proto   |   1 +
 hbase-native-client/if/Client.proto  |   8 +-
 hbase-native-client/if/ClusterStatus.proto   |   3 +
 hbase-native-client/if/Filter.proto  |   1 +
 hbase-native-client/if/Master.proto  |  47 +++-
 hbase-native-client/if/MasterProcedure.proto |  40 +++
 hbase-native-client/if/RSGroup.proto |  34 ++
 hbase-native-client/if/RSGroupAdmin.proto| 136 ++
 hbase-native-client/if/ZooKeeper.proto   |  13 +++
 11 files changed, 304 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/bin/copy-protobuf.sh
--
diff --git a/hbase-native-client/bin/copy-protobuf.sh 
b/hbase-native-client/bin/copy-protobuf.sh
new file mode 100755
index 000..c9e70f2
--- /dev/null
+++ b/hbase-native-client/bin/copy-protobuf.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+BIN_DIR=$(dirname "$0")
+PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
+PB_DEST_DIR="${BIN_DIR}/../if/"
+rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}

http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 725ed6a..9cbd8b7 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -23,22 +23,38 @@ set -x
 eval "$(docker-machine env docker-vm)"
 eval "$(docker-machine env dinghy)"
 
-# Build the image
-docker build -t hbase_native .
+BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd  2>&1 > 
/dev/null)
+BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd  2>&1 > 
/dev/null)
 
+${BIN_DIR}/copy-protobuf.sh
 
+# Go into the base dir. This just makes things cleaner.
+pushd ${BASE_DIR}
+
+# Make sure that there is a thrid-party dir.
 mkdir third-party || true
+
+# Get gtest
+# TODO(eclark): Remove this ( see HBASE-15427 )
 if [[ ! -d third-party/googletest ]]; then
 git clone https://github.com/google/googletest.git 
third-party/googletest
 fi
 
+# We don't want to have to re-download all the jars in docker if we can help it
 if [[ ! -d ~/.m2 ]]; then
 echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
 exit 1
 fi;
 
+# Build the image
+# 
+# This shouldn't be needed after the development environment is a little more 
stable.
+docker build -t hbase_native .
+
+# After the image is built run the thing
 docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-   -v ${PWD}/..:/usr/src/hbase \
+   -v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash
+popd

http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Admin.proto
--
diff --git a/hbase-native-client/if/Admin.proto 
b/hbase-native-client/if/Admin.proto
index e905340..a1905a4 100644
--- a/hbase-native-client/if/Admin.proto
+++ b/hbase-native-client/if/Admin.proto
@@ -25,6 +25,7 @@ option java_generic_services = true;
 option java_generate_equals_and_hash = true;
 option optimize_for = SPEED;
 
+import "Client.proto";
 import "HBase.proto";
 import "WAL.proto";
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Client.proto
--
diff --git a/hbase-native-

[31/50] [abbrv] hbase git commit: HBASE-15821 Document TestUtil

[eclark]

HBASE-15821 Document TestUtil


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8014cdfe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8014cdfe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8014cdfe

Branch: refs/heads/HBASE-14850
Commit: 8014cdfe9d716eae92a91a7f376c002c05446ff9
Parents: dea1eb6
Author: Elliott Clark 
Authored: Fri May 13 09:17:49 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/format-code.sh  |  2 +-
 .../connection/connection-pool.h|  4 ++--
 hbase-native-client/core/location-cache.cc  |  2 +-
 hbase-native-client/core/location-cache.h   |  2 +-
 hbase-native-client/test-util/test-util.cc  | 20 +---
 hbase-native-client/test-util/test-util.h   | 20 +++-
 6 files changed, 33 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/bin/format-code.sh
--
diff --git a/hbase-native-client/bin/format-code.sh 
b/hbase-native-client/bin/format-code.sh
index 36a114d..cc8b368 100755
--- a/hbase-native-client/bin/format-code.sh
+++ b/hbase-native-client/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
 IFS=$'\n\t'
 
 
-find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 
clang-format -i
+find core connection serde utils test-util -name "*.h" -or -name "*.cc" | 
xargs -P8 clang-format -i
 find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/connection/connection-pool.h
--
diff --git a/hbase-native-client/connection/connection-pool.h 
b/hbase-native-client/connection/connection-pool.h
index 605a81b..b8c950b 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -88,8 +88,8 @@ public:
   void Close(const hbase::pb::ServerName );
 
 private:
-  std::shared_ptr GetCached(const hbase::pb::ServerName& sn);
-  std::shared_ptr GetNew(const hbase::pb::ServerName& sn);
+  std::shared_ptr GetCached(const hbase::pb::ServerName );
+  std::shared_ptr GetNew(const hbase::pb::ServerName );
   std::unordered_map
   connections_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index 6ba8add..efd2210 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName , const 
string ) {
   return this->LocateMeta()
   .via(cpu_executor_.get())
   .then([this](ServerName sn) { return this->cp_.Get(sn); })
-.then([tn, row, this](std::shared_ptr service) {
+  .then([tn, row, this](std::shared_ptr service) {
 return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
   })
   .then([this](Response resp) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/core/location-cache.h
--
diff --git a/hbase-native-client/core/location-cache.h 
b/hbase-native-client/core/location-cache.h
index d435530..830cd96 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -55,7 +55,7 @@ public:
* @param io_executor executor used to talk to the network
*/
   LocationCache(std::string quorum_spec,
-std::shared_ptr cpu_exector,
+std::shared_ptr cpu_executor,
 std::shared_ptr io_executor);
   /**
* Destructor.

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/test-util/test-util.cc
--
diff --git a/hbase-native-client/test-util/test-util.cc 
b/hbase-native-client/test-util/test-util.cc
index e5fba48..88ce7c8 100644
--- a/hbase-native-client/test-util/test-util.cc
+++ b/hbase-native-client/test-util/test-util.cc
@@ -24,13 +24,18 @@
 using hbase::TestUtil;
 using folly::Random;
 
-const static int STR_LEN = 32;
-
-std::string TestUtil::RandString() {
-  auto s = std::string(STR_LEN, 'z');
-
-  for (int i = 0; i < STR_LEN; i++) {
+std::string TestUtil::RandString(int len) {
+  // Create the whole string.
+  // 

[05/50] [abbrv] hbase git commit: HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

[eclark]

HBASE-14548 Expand how table coprocessor jar and dependency path can be 
specified (Xiang Li)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63296978
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63296978
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63296978

Branch: refs/heads/HBASE-14850
Commit: 632969787aa9e9e1d73b83449b93e21f91110b73
Parents: 496fd98
Author: Jerry He 
Authored: Sat Jul 9 18:00:41 2016 -0700
Committer: Jerry He 
Committed: Sat Jul 9 18:01:49 2016 -0700

--
 .../hbase/util/CoprocessorClassLoader.java  | 72 +---
 .../hbase/util/TestCoprocessorClassLoader.java  | 43 
 src/main/asciidoc/_chapters/cp.adoc |  9 ++-
 3 files changed, 97 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 11016c3..c3635cb 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase 
{
 super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
   Configuration conf) throws IOException {
 // Copy the jar to the local filesystem
 String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends 
ClassLoaderBase {
   }
 }
 
-FileSystem fs = path.getFileSystem(conf);
-File dst = new File(parentDirStr, "." + pathPrefix + "."
-  + path.getName() + "." + System.currentTimeMillis() + ".jar");
-fs.copyToLocalFile(path, new Path(dst.toString()));
-dst.deleteOnExit();
-
-addURL(dst.getCanonicalFile().toURI().toURL());
-
-JarFile jarFile = new JarFile(dst.toString());
-try {
-  Enumeration entries = jarFile.entries();
-  while (entries.hasMoreElements()) {
-JarEntry entry = entries.nextElement();
-Matcher m = libJarPattern.matcher(entry.getName());
-if (m.matches()) {
-  File file = new File(parentDirStr, "." + pathPrefix + "."
-+ path.getName() + "." + System.currentTimeMillis() + "." + 
m.group(1));
-  try (FileOutputStream outStream = new FileOutputStream(file)) {
-IOUtils.copyBytes(jarFile.getInputStream(entry),
-  outStream, conf, true);
+FileSystem fs = pathPattern.getFileSystem(conf);
+Path pathPattern1 = fs.isDirectory(pathPattern) ?
+  new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a 
directory is specified
+FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all 
files that match the pattern
+if (fileStatuses == null || fileStatuses.length == 0) {  // if no one 
matches
+  throw new FileNotFoundException(pathPattern1.toString());
+} else {
+  boolean validFileEncountered = false;
+  for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file 
that match the pattern
+if (fs.isFile(path)) {  // only process files, skip for directories
+  File dst = new File(parentDirStr, "." + pathPrefix + "."
++ path.getName() + "." + System.currentTimeMillis() + ".jar");
+  fs.copyToLocalFile(path, new Path(dst.toString()));
+  dst.deleteOnExit();
+
+  addURL(dst.getCanonicalFile().toURI().toURL());
+
+  JarFile jarFile = new JarFile(dst.toString());
+  try {
+Enumeration entries = jarFile.entries();  // get entries 
inside a jar file
+while (entries.hasMoreElements()) {
+  JarEntry entry = entries.nextElement();
+  Matcher m = libJarPattern.matcher(entry.getName());
+  if (m.matches()) {
+File file = new 

[17/50] [abbrv] hbase git commit: HBASE-14852 Update build env

[eclark]

HBASE-14852 Update build env

Also includes HBASE-14858 Clean up so core is ready for development on a recent 
version of c++


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8556e3a3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8556e3a3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8556e3a3

Branch: refs/heads/HBASE-14850
Commit: 8556e3a348fb4e4d1610ab44084b70631331a98a
Parents: a396ae7
Author: Elliott Clark 
Authored: Thu Nov 19 16:28:11 2015 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/.buckconfig |   2 +
 hbase-native-client/.gitignore  |  23 +--
 hbase-native-client/CMakeLists.txt  | 157 ---
 hbase-native-client/Dockerfile  |  22 +++
 hbase-native-client/README.md   |  15 +-
 hbase-native-client/bin/build-all.sh|  41 -
 hbase-native-client/bin/build-thirdparty.sh |  64 
 hbase-native-client/bin/download-thirdparty.sh  |  70 -
 hbase-native-client/bin/hbase-client-env.sh |  47 --
 hbase-native-client/bin/start-docker.sh |  32 
 .../cmake_modules/FindGTest.cmake   |  53 ---
 .../cmake_modules/FindLibEv.cmake   |  47 --
 hbase-native-client/core/BUCK   |  50 ++
 hbase-native-client/core/admin.cc   |  20 +++
 hbase-native-client/core/admin.h|  22 +++
 hbase-native-client/core/client.cc  |  38 +
 hbase-native-client/core/client.h   |  24 +++
 hbase-native-client/core/connection.cc  |  20 +++
 hbase-native-client/core/connection.h   |  26 +++
 hbase-native-client/core/connection_attr.h  |  24 +++
 hbase-native-client/core/delete.cc  |  21 +++
 hbase-native-client/core/delete.h   |  27 
 hbase-native-client/core/get.cc |  20 +++
 hbase-native-client/core/get.h  |  22 +++
 hbase-native-client/core/hbase_macros.h |  56 +++
 hbase-native-client/core/mutation.cc|  41 +
 hbase-native-client/core/mutation.h |  58 +++
 hbase-native-client/core/put.cc |  21 +++
 hbase-native-client/core/put.h  |  27 
 hbase-native-client/core/scanner.cc |  20 +++
 hbase-native-client/core/scanner.h  |  22 +++
 hbase-native-client/rpc/CMakeLists.txt  |  17 ++
 hbase-native-client/src/async/CMakeLists.txt|  32 
 hbase-native-client/src/async/get-test.cc   |  59 ---
 hbase-native-client/src/async/hbase_admin.cc|  57 ---
 hbase-native-client/src/async/hbase_admin.h |  69 
 hbase-native-client/src/async/hbase_client.cc   |  47 --
 hbase-native-client/src/async/hbase_client.h|  60 ---
 .../src/async/hbase_connection.cc   |  37 -
 .../src/async/hbase_connection.h|  52 --
 hbase-native-client/src/async/hbase_errno.h |  23 ---
 hbase-native-client/src/async/hbase_get.cc  |  61 ---
 hbase-native-client/src/async/hbase_get.h   |  73 -
 .../src/async/hbase_mutations.cc| 111 -
 hbase-native-client/src/async/hbase_mutations.h | 119 --
 hbase-native-client/src/async/hbase_result.cc   |  37 -
 hbase-native-client/src/async/hbase_result.h|  44 --
 hbase-native-client/src/async/hbase_scanner.cc  |  59 ---
 hbase-native-client/src/async/hbase_scanner.h   |  80 --
 hbase-native-client/src/async/mutations-test.cc | 102 
 hbase-native-client/src/core/CMakeLists.txt |  31 
 hbase-native-client/src/core/admin.cc   |  20 ---
 hbase-native-client/src/core/admin.h|  25 ---
 hbase-native-client/src/core/client.cc  |  20 ---
 hbase-native-client/src/core/client.h   |  25 ---
 hbase-native-client/src/core/connection.cc  |  22 ---
 hbase-native-client/src/core/connection.h   |  26 ---
 hbase-native-client/src/core/connection_attr.h  |  30 
 hbase-native-client/src/core/delete.cc  |  22 ---
 hbase-native-client/src/core/delete.h   |  29 
 hbase-native-client/src/core/get.cc |  20 ---
 hbase-native-client/src/core/get.h  |  26 ---
 .../src/core/hbase_connection_attr.cc   |  41 -
 .../src/core/hbase_connection_attr.h|  51 --
 hbase-native-client/src/core/hbase_macros.h |  60 ---
 hbase-native-client/src/core/hbase_types.h  |  83 --
 hbase-native-client/src/core/mutation.cc|  42 -
 hbase-native-client/src/core/mutation.h |  48 --
 hbase-native-client/src/core/put.cc |  22 ---
 hbase-native-client/src/core/put.h

[28/50] [abbrv] hbase git commit: HBASE-15620 Add on Call serialization

[eclark]

tp://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include 
+
+#include 
+
+#include 
+
+#include "serde/client-serializer.h"
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace folly::io;
+
+TEST(ClientSerializerTest, PreambleIncludesHBas) {
+  ClientSerializer ser;
+  auto buf = ser.preamble();
+  const char *p = reinterpret_cast(buf->data());
+  // Take the first for chars and make sure they are the
+  // magic string
+  EXPECT_EQ("HBas", std::string(p, 4));
+
+  EXPECT_EQ(6, buf->computeChainDataLength());
+}
+
+TEST(ClientSerializerTest, PreambleIncludesVersion) {
+  ClientSerializer ser;
+  auto buf = ser.preamble();
+  EXPECT_EQ(0, static_cast(buf->data())[4]);
+  EXPECT_EQ(80, static_cast(buf->data())[5]);
+}
+
+TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
+  ClientSerializer ser;
+  auto header = ser.header("elliott");
+
+  // The header should be prefixed by 4 bytes of length.
+  EXPECT_EQ(4, header->length());
+  EXPECT_TRUE(header->length() < header->computeChainDataLength());
+  EXPECT_TRUE(header->isChained());
+
+  // Now make sure the length is correct.
+  Cursor cursor(header.get());
+  auto prefixed_len = cursor.readBE();
+  EXPECT_EQ(prefixed_len, header->next()->length());
+}
+
+TEST(ClientSerializerTest, TestHeaderDecode) {
+  ClientSerializer ser;
+  auto buf = ser.header("elliott");
+  auto header_buf = buf->next();
+  ConnectionHeader h;
+
+  EXPECT_TRUE(h.ParseFromArray(header_buf->data(), header_buf->length()));
+  EXPECT_EQ("elliott", h.user_info().effective_user());
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-serializer.cc
--
diff --git a/hbase-native-client/serde/client-serializer.cc 
b/hbase-native-client/serde/client-serializer.cc
new file mode 100644
index 000..881b6e4
--- /dev/null
+++ b/hbase-native-client/serde/client-serializer.cc
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "serde/client-serializer.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+
+using folly::IOBuf;
+using folly::io::RWPrivateCursor;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayOutputStream;
+using google::protobuf::io::CodedOutputStream;
+using google::protobuf::io::ZeroCopyOutputStream;
+using std::string;
+using std::unique_ptr;
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t DEFAULT_AUTH_TYPE = 80;
+
+ClientSerializer::ClientSerializer() : auth_type_(DEFAULT_AUTH_TYPE) {}
+
+unique_ptr ClientSerializer::preamble() {
+  auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
+  magic->append(2);
+  RWPrivateCursor c(magic.get());
+  c.skip(4);
+  // Version
+  c.write(RPC_VERSION);
+  // Standard security aka Please don't lie to me.
+  c.write(auth_type_);
+  return magic;
+}
+
+unique_ptr ClientSerializer::header(const string ) {
+  pb::ConnectionHeader h;
+
+  // TODO(eclark): Make this not a total lie.
+  h.mutable_user_info()->set_effective_user(user);
+  // The service name that we want to talk to.
+  //
+  // Right now we're completely ignoring the service interface.
+  // That may or may not be the correct thing to do.
+  // It worked for a while with the java client; until it
+  // didn't.
+  h.set_service_name(INTERFACE);
+  return prepend_length(serialize_message(h));
+}
+
+unique_ptr ClientSerializer::request(const uint32_t call_id,
+

[24/50] [abbrv] hbase git commit: HBASE-15418 Clean up un-used warning in test util

[eclark]

HBASE-15418 Clean up un-used warning in test util


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/45cf5657
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/45cf5657
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/45cf5657

Branch: refs/heads/HBASE-14850
Commit: 45cf5657acc4cc7a61655fa6b76eb90910fabb21
Parents: 26ba4ed
Author: Elliott Clark 
Authored: Fri Mar 25 15:44:06 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../bin/start_local_hbase_and_wait.sh   |  9 +-
 .../bin/stop_local_hbase_and_wait.sh|  2 +-
 hbase-native-client/core/BUCK   |  6 
 .../core/native-client-test-env.cc  |  9 --
 hbase-native-client/core/test_env.h | 32 
 5 files changed, 15 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/bin/start_local_hbase_and_wait.sh
--
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh 
b/hbase-native-client/bin/start_local_hbase_and_wait.sh
index 64d0b68..cfc71f9 100755
--- a/hbase-native-client/bin/start_local_hbase_and_wait.sh
+++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
@@ -17,10 +17,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Clean up from any other tests.
+rm -rf /tmp/hbase-*
+
+# Start the master/regionservers.
 $PWD/../bin/start-hbase.sh
 
-until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == 
"200" ]
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" 
http://localhost:16010/jmx) == "200" ]
 do
  printf "Waiting for local HBase cluster to start\n"
  sleep 1
 done
+
+# This sucks, but master can easily be up and meta not be assigned yet.
+sleep 30

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/bin/stop_local_hbase_and_wait.sh
--
diff --git a/hbase-native-client/bin/stop_local_hbase_and_wait.sh 
b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
index 4e89334..761412a 100755
--- a/hbase-native-client/bin/stop_local_hbase_and_wait.sh
+++ b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
@@ -17,7 +17,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-$PWD/../bin/stop-hbase.sh
+ps aux | grep proc_master | awk '{print $2}' | xargs kill -9
 
 while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == 
"200" ]
 do

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 817b5a0..d1e89d1 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -51,9 +51,6 @@ cxx_library(name="core",
 ], )
 
 cxx_test(name="simple-test",
- headers=[
- "test_env.h",
- ],
  srcs=[
  "native-client-test-env.cc",
  "simple-native-client-test.cc",
@@ -63,9 +60,6 @@ cxx_test(name="simple-test",
  ],
  run_test_separately=True, )
 cxx_test(name="location-cache-test",
- headers=[
- "test_env.h",
- ],
  srcs=[
  "native-client-test-env.cc",
  "location-cache-test.cc",

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/native-client-test-env.cc
--
diff --git a/hbase-native-client/core/native-client-test-env.cc 
b/hbase-native-client/core/native-client-test-env.cc
index a86961f..07f30a6 100644
--- a/hbase-native-client/core/native-client-test-env.cc
+++ b/hbase-native-client/core/native-client-test-env.cc
@@ -18,18 +18,21 @@
  */
 
 #include 
-#include 
 
 namespace {
 
 class NativeClientTestEnv : public ::testing::Environment {
  public:
   void SetUp() override {
-init_test_env();
+// start local HBase cluster to be reused by all tests
+auto result = system("bin/start_local_hbase_and_wait.sh");
+ASSERT_EQ(0, result);
   }
 
   void TearDown() override {
-clean_test_env();
+// shutdown local HBase cluster
+auto result = system("bin/stop_local_hbase_and_wait.sh");
+ASSERT_EQ(0, result);
   }
 };
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/test_env.h
--
diff --git a/hbase-native-client/core/test_env.h 

[15/50] [abbrv] hbase git commit: HBASE-15739 Add region location lookup from meta

[eclark]

 sn, this);
+auto wrapped = std::make_shared(new_con, sn, *this);
 connections_[sn] = wrapped;
 return new_con;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/pipeline.cc
--
diff --git a/hbase-native-client/connection/pipeline.cc 
b/hbase-native-client/connection/pipeline.cc
index b9f5e0b..39cce8b 100644
--- a/hbase-native-client/connection/pipeline.cc
+++ b/hbase-native-client/connection/pipeline.cc
@@ -30,6 +30,8 @@ using namespace folly;
 using namespace hbase;
 using namespace wangle;
 
+RpcPipelineFactory::RpcPipelineFactory() : user_util_() {}
+
 SerializePipeline::Ptr
 RpcPipelineFactory::newPipeline(std::shared_ptr sock) {
   auto pipeline = SerializePipeline::create();

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/pipeline.h
--
diff --git a/hbase-native-client/connection/pipeline.h 
b/hbase-native-client/connection/pipeline.h
index 6c4f4ff..e90cb93 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -31,6 +31,7 @@ using SerializePipeline =
 
 class RpcPipelineFactory : public wangle::PipelineFactory {
 public:
+  RpcPipelineFactory();
   SerializePipeline::Ptr
   newPipeline(std::shared_ptr sock) override;
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/response.h
--
diff --git a/hbase-native-client/connection/response.h 
b/hbase-native-client/connection/response.h
index a7f7939..d2eb19a 100644
--- a/hbase-native-client/connection/response.h
+++ b/hbase-native-client/connection/response.h
@@ -31,10 +31,12 @@ namespace hbase {
 
 class Response {
 public:
-  Response() : call_id_(0) {}
+  Response() : call_id_(0), response_(nullptr) {}
   uint32_t call_id() { return call_id_; }
   void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-  std::shared_ptr response() { return response_; }
+  std::shared_ptr response() const {
+return response_;
+  }
   void set_response(std::shared_ptr response) {
 response_ = std::move(response);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index e555ba4..447248b 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -22,6 +22,7 @@ cxx_library(
 "client.h",
 "connection.h",
 "hbase_macros.h",
+"region-location.h",
 "location-cache.h",
 "table-name.h",
 # TODO: move this out of exported
@@ -32,7 +33,6 @@ cxx_library(
 "client.cc",
 "location-cache.cc",
 "meta-utils.cc",
-"table-name.cc",
 ],
 deps=[
 "//connection:connection",
@@ -42,6 +42,7 @@ cxx_library(
 "//third-party:wangle",
 "//third-party:zookeeper_mt",
 ],
+compiler_flags=['-Weffc++'],
 visibility=[
 'PUBLIC',
 ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index c81deba..539051a 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -21,20 +21,31 @@
 #include 
 #include 
 
+#include "connection/response.h"
+#include "if/Client.pb.h"
 #include "if/ZooKeeper.pb.h"
 #include "serde/zk-deserializer.h"
 
 using namespace std;
 using namespace folly;
-using namespace hbase::pb;
-using namespace hbase;
+
+using hbase::Response;
+using hbase::LocationCache;
+using hbase::RegionLocation;
+using hbase::HBaseService;
+using hbase::pb::ScanResponse;
+using hbase::pb::TableName;
+using hbase::pb::ServerName;
+using hbase::pb::MetaRegionServer;
+using hbase::pb::RegionInfo;
 
 // TODO(eclark): make this configurable on client creation
 static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
 
 LocationCache::LocationCache(string quorum_spec,
  shared_ptr executor)
-: quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
+: quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
+  meta_lock_(), cp_(), meta_util_() {
   zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
 }
 
@@ -89,3 +100,23 @@ ServerName LocationCache::ReadMetaLocation() {
   }
   return mrs.server();
 }
+
+Future LocationCa

[08/50] [abbrv] hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells, addendum

[eclark]

HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells, addendum


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/356c1307
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/356c1307
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/356c1307

Branch: refs/heads/HBASE-14850
Commit: 356c1307021e931e871307077f71aab79dc43def
Parents: 0875c35
Author: Yu Li 
Authored: Mon Jul 11 22:43:26 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 22:44:32 2016 +0800

--
 .../hadoop/hbase/regionserver/AbstractMemStore.java   | 14 +++---
 .../hadoop/hbase/regionserver/MemStoreCompactor.java  |  4 ++--
 .../hadoop/hbase/regionserver/MutableSegment.java |  8 
 .../org/apache/hadoop/hbase/regionserver/Segment.java |  6 +++---
 4 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 79e95af..2b9910f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,8 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-boolean useMSLAB = (toAdd != cell);
-return internalAdd(toAdd, useMSLAB);
+boolean mslabUsed = (toAdd != cell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -157,8 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long delete(Cell deleteCell) {
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-boolean useMSLAB = (toAdd != deleteCell);
-long s = internalAdd(toAdd, useMSLAB);
+boolean mslabUsed = (toAdd != deleteCell);
+long s = internalAdd(toAdd, mslabUsed);
 return s;
   }
 
@@ -390,11 +390,11 @@ public abstract class AbstractMemStore implements 
MemStore {
*
* Callers should ensure they already have the read lock taken
* @param toAdd the cell to add
-   * @param useMSLAB whether using MSLAB
+   * @param mslabUsed whether using MSLAB
* @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
-long s = active.add(toAdd, useMSLAB);
+  private long internalAdd(final Cell toAdd, final boolean mslabUsed) {
+long s = active.add(toAdd, mslabUsed);
 setOldestEditTimeToNow();
 checkActiveSize();
 return s;

http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
index 23f792b..042de0a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
@@ -197,8 +197,8 @@ class MemStoreCompactor {
   // The scanner is doing all the elimination logic
   // now we just copy it to the new segment
   Cell newKV = result.maybeCloneWithAllocator(c);
-  boolean useMSLAB = (newKV != c);
-  result.internalAdd(newKV, useMSLAB);
+  boolean mslabUsed = (newKV != c);
+  result.internalAdd(newKV, mslabUsed);
 
 }
 kvs.clear();

http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index 6e54060..e62249a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -36,11 +36,11 @@ public class MutableSegment extends Segment {
   /**
* Adds the given cell into the segment
* @param cell the cell to add
-   * @param useMSLAB whether using MSLAB
+   * @param mslabUsed 

[19/50] [abbrv] hbase git commit: HBASE-14853 Add on protobuf to c++ chain

[eclark]

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Master.proto
--
diff --git a/hbase-native-client/if/Master.proto 
b/hbase-native-client/if/Master.proto
new file mode 100644
index 000..4d3a2e1
--- /dev/null
+++ b/hbase-native-client/if/Master.proto
@@ -0,0 +1,778 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// All to do with the Master.  Includes schema management since these
+// changes are run by the Master process.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+import "ClusterStatus.proto";
+import "ErrorHandling.proto";
+import "Procedure.proto";
+import "Quota.proto";
+
+/* Column-level protobufs */
+
+message AddColumnRequest {
+  required TableName table_name = 1;
+  required ColumnFamilySchema column_families = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message AddColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DeleteColumnRequest {
+  required TableName table_name = 1;
+  required bytes column_name = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message DeleteColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyColumnRequest {
+  required TableName table_name = 1;
+  required ColumnFamilySchema column_families = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+/* Region-level Protos */
+
+message MoveRegionRequest {
+  required RegionSpecifier region = 1;
+  optional ServerName dest_server_name = 2;
+}
+
+message MoveRegionResponse {
+}
+
+/**
+ * Dispatch merging the specified regions.
+ */
+message DispatchMergingRegionsRequest {
+  required RegionSpecifier region_a = 1;
+  required RegionSpecifier region_b = 2;
+  optional bool forcible = 3 [default = false];
+}
+
+message DispatchMergingRegionsResponse {
+}
+
+message AssignRegionRequest {
+  required RegionSpecifier region = 1;
+}
+
+message AssignRegionResponse {
+}
+
+message UnassignRegionRequest {
+  required RegionSpecifier region = 1;
+  optional bool force = 2 [default = false];
+}
+
+message UnassignRegionResponse {
+}
+
+message OfflineRegionRequest {
+  required RegionSpecifier region = 1;
+}
+
+message OfflineRegionResponse {
+}
+
+/* Table-level protobufs */
+
+message CreateTableRequest {
+  required TableSchema table_schema = 1;
+  repeated bytes split_keys = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message CreateTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DeleteTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message TruncateTableRequest {
+  required TableName tableName = 1;
+  optional bool preserveSplits = 2 [default = false];
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message TruncateTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message EnableTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message EnableTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DisableTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message DisableTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyTableRequest {
+  required TableName table_name = 1;
+  required TableSchema table_schema = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyTableResponse 

[07/50] [abbrv] hbase git commit: HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)

[eclark]

HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0875c357
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0875c357
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0875c357

Branch: refs/heads/HBASE-14850
Commit: 0875c35781f1fcc06d38308b915a01791c7a6a20
Parents: 9cf012c
Author: tedyu 
Authored: Sun Jul 10 23:28:32 2016 -0700
Committer: tedyu 
Committed: Sun Jul 10 23:28:32 2016 -0700

--
 .../hadoop/hbase/replication/TestReplicationSmallTests.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0875c357/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 8efa67e..e3cc6e9 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -732,7 +732,9 @@ public class TestReplicationSmallTests extends 
TestReplicationBase {
 
   ClusterStatus status = admin.getClusterStatus();
 
-  for (ServerName server : status.getServers()) {
+  for (JVMClusterUtil.RegionServerThread thread :
+  utility1.getHBaseCluster().getRegionServerThreads()) {
+ServerName server = thread.getRegionServer().getServerName();
 ServerLoad sl = status.getLoad(server);
 List rLoadSourceList = 
sl.getReplicationLoadSourceList();
 ReplicationLoadSink rLoadSink = sl.getReplicationLoadSink();

[12/50] [abbrv] hbase git commit: HBASE-15766 Show working puts

[eclark]

 std::string ) const;
 };
 } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/region-location.h
--
diff --git a/hbase-native-client/core/region-location.h 
b/hbase-native-client/core/region-location.h
index 7922c95..7887526 100644
--- a/hbase-native-client/core/region-location.h
+++ b/hbase-native-client/core/region-location.h
@@ -27,15 +27,19 @@ namespace hbase {
 
 class RegionLocation {
 public:
-  RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
+  RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
+ hbase::pb::ServerName sn,
  std::shared_ptr service)
-  : ri_(ri), sn_(sn), service_(service) {}
+  : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
 
   const hbase::pb::RegionInfo _info() { return ri_; }
   const hbase::pb::ServerName _name() { return sn_; }
+  const std::string _name() { return region_name_; }
   std::shared_ptr service() { return service_; }
+  void set_service(std::shared_ptr s) { service_ = s; }
 
 private:
+  std::string region_name_;
   hbase::pb::RegionInfo ri_;
   hbase::pb::ServerName sn_;
   std::shared_ptr service_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/simple-client.cc
--
diff --git a/hbase-native-client/core/simple-client.cc 
b/hbase-native-client/core/simple-client.cc
index 00e3369..39c82c3 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -19,16 +19,21 @@
 
 #include 
 #include 
+#include 
 #include 
+#include 
 #include 
 
+#include 
 #include 
 #include 
+#include 
 
 #include "connection/connection-pool.h"
 #include "core/client.h"
 #include "if/Client.pb.h"
 #include "if/ZooKeeper.pb.h"
+#include "serde/server-name.h"
 #include "serde/table-name.h"
 
 using namespace folly;
@@ -39,16 +44,41 @@ using hbase::Request;
 using hbase::HBaseService;
 using hbase::LocationCache;
 using hbase::ConnectionPool;
+using hbase::ConnectionFactory;
 using hbase::pb::TableName;
 using hbase::pb::ServerName;
 using hbase::pb::RegionSpecifier_RegionSpecifierType;
-using hbase::pb::GetRequest;
-using hbase::pb::GetResponse;
+using hbase::pb::MutateRequest;
+using hbase::pb::MutationProto_MutationType;
 
 // TODO(eclark): remove the need for this.
 DEFINE_string(table, "t", "What region to send a get");
 DEFINE_string(row, "test", "What row to get");
 DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
+DEFINE_uint64(columns, 1, "How many columns to write");
+DEFINE_int32(threads, 6, "How many cpu threads");
+
+std::unique_ptr MakeRequest(uint64_t col, std::string region_name) {
+  auto req = Request::mutate();
+  auto msg = std::static_pointer_cast(req->req_msg());
+  auto region = msg->mutable_region();
+  auto suf = folly::to(col);
+
+  region->set_value(region_name);
+  region->set_type(RegionSpecifier_RegionSpecifierType::
+   RegionSpecifier_RegionSpecifierType_REGION_NAME);
+  auto mutation = msg->mutable_mutation();
+  mutation->set_row(FLAGS_row + suf);
+  mutation->set_mutate_type(
+  MutationProto_MutationType::MutationProto_MutationType_PUT);
+  auto column = mutation->add_column_value();
+  column->set_family("d");
+  auto qual = column->add_qualifier_value();
+  qual->set_qualifier(suf);
+  qual->set_value(".");
+
+  return std::move(req);
+}
 
 int main(int argc, char *argv[]) {
   google::SetUsageMessage(
@@ -56,13 +86,41 @@ int main(int argc, char *argv[]) {
   google::ParseCommandLineFlags(, , true);
   google::InitGoogleLogging(argv[0]);
 
-  // Create a connection factory
-  ConnectionPool cp;
-  auto cpu_ex = wangle::getCPUExecutor();
-  LocationCache cache{FLAGS_zookeeper, cpu_ex};
-  auto result =
-  cache.LocateFromMeta(folly::to(FLAGS_table), FLAGS_row)
-  .get(milliseconds(5000));
+  // Set up thread pools.
+  auto cpu_pool =
+  std::make_shared(FLAGS_threads);
+  wangle::setCPUExecutor(cpu_pool);
+  auto io_pool = std::make_shared(5);
+  wangle::setIOExecutor(io_pool);
+
+  // Create the cache.
+  LocationCache cache{FLAGS_zookeeper, cpu_pool};
+
+  auto row = FLAGS_row;
+  auto tn = folly::to(FLAGS_table);
+
+  auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(5000));
+  auto connection = loc->service();
+
+  auto num_puts = FLAGS_columns;
+
+  auto results = std::vector<Future>{};
+  uint64_t col{0};
+  for (; col < num_puts; col++) {
+results.push_back(folly::makeFuture(col)
+  .via(cpu_pool.get())
+  .then([loc](uint64_t col) {
+   

[09/50] [abbrv] hbase git commit: HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)

[eclark]

HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b5fbf8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b5fbf8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b5fbf8d

Branch: refs/heads/HBASE-14850
Commit: 3b5fbf8d73b0ee50a0b04b044f93c1af39e5d79d
Parents: 356c130
Author: Gary Helmling 
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling 
Committed: Mon Jul 11 10:33:22 2016 -0700

--
 .../apache/hadoop/hbase/ipc/BufferChain.java|   5 +-
 .../hbase/security/token/SecureTestCluster.java | 123 +++
 .../TestDelegationTokenWithEncryption.java  |  91 ++
 .../token/TestGenerateDelegationToken.java  |  93 +-
 4 files changed, 218 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index 39efa40..7adc94d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -63,8 +63,9 @@ class BufferChain {
 byte [] bytes = new byte [this.remaining];
 int offset = 0;
 for (ByteBuffer bb: this.buffers) {
-  System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, 
bb.limit());
-  offset += bb.capacity();
+  int length = bb.remaining();
+  bb.get(bytes, offset, length);
+  offset += length;
 }
 return bytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 000..a469537
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+  protected static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+
+  protected static String USERNAME;
+
+  private static LocalHBaseCluster CLUSTER;
+
+  private static final File KEYTAB_FILE = new 
File(TEST_UTIL.getDataTestDir("keytab").toUri()
+  .getPath());
+  private static MiniKdc KDC;
+
+  private static String HOST = "localhost";
+
+  private static String PRINCIPAL;
+
+  private static String HTTP_PRINCIPAL;
+
+  /**
+   * Setup the security configuration for hdfs.
+   */
+  private static void setHdfsSecuredConfiguration(Configuration conf) throws 
Exception {
+// change XXX_USER_NAME_KEY to XXX_KERBEROS_PRINCIPAL_KEY after we 

[18/50] [abbrv] hbase git commit: HBASE-15078 Added ability to start/stop hbase local cluster for tests, global test_env for gtest, small changes to dockerfile and docker run.

[eclark]

HBASE-15078 Added ability to start/stop hbase local cluster for tests, global 
test_env for gtest, small changes to dockerfile and docker run.

Added check ~/.m2 folder exists; moved scripts to ./bin


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5431
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5431
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5431

Branch: refs/heads/HBASE-14850
Commit: 54314e3f1be9d6b5efcab845b8f321e4b4e2
Parents: 8556e3a
Author: Mikhail Antonov 
Authored: Wed Jan 6 15:08:21 2016 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/.buckconfig |  2 +-
 hbase-native-client/Dockerfile  |  2 +-
 hbase-native-client/bin/start-docker.sh | 10 -
 .../bin/start_local_hbase_and_wait.sh   | 26 
 .../bin/stop_local_hbase_and_wait.sh| 26 
 hbase-native-client/core/BUCK   | 15 +++
 .../core/HBaseNativeClientTestEnv.cc| 42 
 .../core/SampleNativeClientTest.cc  | 28 +
 hbase-native-client/core/test_env.h | 30 ++
 9 files changed, 178 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/.buckconfig
--
diff --git a/hbase-native-client/.buckconfig b/hbase-native-client/.buckconfig
index 3227a2a..402ef27 100644
--- a/hbase-native-client/.buckconfig
+++ b/hbase-native-client/.buckconfig
@@ -1,2 +1,2 @@
 [cxx]
-  gtest_dep = //third-party/googletest/googletest:google-test
+  gtest_dep = //third-party:google-test

http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 70e823b..5f17f04 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -17,6 +17,6 @@
 
 FROM pjameson/buck-folly-watchman
 
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim 
maven inetutils-ping
 
 WORKDIR /usr/local/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 1c9b02e..bf38912 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -28,5 +28,13 @@ if [[ ! -d third-party/googletest ]]; then
 git clone https://github.com/google/googletest.git 
third-party/googletest
 fi
 
+if [[ ! -d ~/.m2 ]]; then
+echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
+exit 1
+fi;
 
-docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native  /bin/bash
+docker run -p 16010:16010/tcp \
+   -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
+   -v ${PWD}/..:/usr/local/src/hbase \
+   -v ~/.m2:/root/.m2 \
+   -it hbase_native  /bin/bash

http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/bin/start_local_hbase_and_wait.sh
--
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh 
b/hbase-native-client/bin/start_local_hbase_and_wait.sh
new file mode 100755
index 000..64d0b68
--- /dev/null
+++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$PWD/../bin/start-hbase.sh
+
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == 
"200" ]
+do
+ printf "Waiting for local HBase 

[23/50] [abbrv] hbase git commit: HBASE-15649 Ignore buck-out when building docker image

[eclark]

HBASE-15649 Ignore buck-out when building docker image


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/098028c5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/098028c5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/098028c5

Branch: refs/heads/HBASE-14850
Commit: 098028c5c61ae7d318dde73fece2c0758f28d113
Parents: bfabc71
Author: Elliott Clark 
Authored: Wed Apr 13 17:02:38 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/.dockerignore | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/098028c5/hbase-native-client/.dockerignore
--
diff --git a/hbase-native-client/.dockerignore 
b/hbase-native-client/.dockerignore
new file mode 100644
index 000..2cfffb7
--- /dev/null
+++ b/hbase-native-client/.dockerignore
@@ -0,0 +1,2 @@
+buck-out
+.buckd

[03/50] [abbrv] hbase git commit: HBASE-16201 NPE in RpcServer causing intermittent UT failure of TestMasterReplication#testHFileCyclicReplication

[eclark]

HBASE-16201 NPE in RpcServer causing intermittent UT failure of 
TestMasterReplication#testHFileCyclicReplication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3c39cbd9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3c39cbd9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3c39cbd9

Branch: refs/heads/HBASE-14850
Commit: 3c39cbd92c3f309c98ca01bfb70ca89bc046a228
Parents: 1578a04
Author: Yu Li 
Authored: Sat Jul 9 01:05:10 2016 +0800
Committer: Yu Li 
Committed: Sat Jul 9 02:02:00 2016 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3c39cbd9/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index ad88e8e..73226aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -2246,7 +2246,13 @@ public class RpcServer implements RpcServerInterface, 
ConfigurationObserver {
   // The above callBlockingMethod will always return a SE.  Strip the SE 
wrapper before
   // putting it on the wire.  Its needed to adhere to the pb Service 
Interface but we don't
   // need to pass it over the wire.
-  if (e instanceof ServiceException) e = e.getCause();
+  if (e instanceof ServiceException) {
+if (e.getCause() == null) {
+  LOG.debug("Caught a ServiceException with null cause", e);
+} else {
+  e = e.getCause();
+}
+  }
 
   // increment the number of requests that were exceptions.
   metrics.exception(e);

[02/50] [abbrv] hbase git commit: Revert "NPE in RpcServer causing intermittent UT failure of TestMasterReplication#testHFileCyclicReplication"

[eclark]

Revert "NPE in RpcServer causing intermittent UT failure of 
TestMasterReplication#testHFileCyclicReplication"

This reverts commit a33097e067b73be8e877b822afa90b89a0c7974f.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1578a045
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1578a045
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1578a045

Branch: refs/heads/HBASE-14850
Commit: 1578a045b6af6e8cd4c85d7b43397e204345a99d
Parents: 5e0d97e
Author: Yu Li 
Authored: Sat Jul 9 02:01:43 2016 +0800
Committer: Yu Li 
Committed: Sat Jul 9 02:01:43 2016 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 8 +---
 1 file changed, 1 insertion(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1578a045/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 73226aa..ad88e8e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -2246,13 +2246,7 @@ public class RpcServer implements RpcServerInterface, 
ConfigurationObserver {
   // The above callBlockingMethod will always return a SE.  Strip the SE 
wrapper before
   // putting it on the wire.  Its needed to adhere to the pb Service 
Interface but we don't
   // need to pass it over the wire.
-  if (e instanceof ServiceException) {
-if (e.getCause() == null) {
-  LOG.debug("Caught a ServiceException with null cause", e);
-} else {
-  e = e.getCause();
-}
-  }
+  if (e instanceof ServiceException) e = e.getCause();
 
   // increment the number of requests that were exceptions.
   metrics.exception(e);

hbase git commit: HBASE-15935 Set up a concurrent walker that walks flushed circular linked lists as a Loop Mode

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 ed07a38d5 -> 0d8e7b2f6


HBASE-15935 Set up a concurrent walker that walks flushed circular linked lists 
as a Loop Mode

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d8e7b2f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d8e7b2f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d8e7b2f

Branch: refs/heads/branch-1
Commit: 0d8e7b2f631b5b5bd9d0c1a4d202b6178bfd1b72
Parents: ed07a38
Author: Joseph Hwang 
Authored: Thu Jun 2 18:00:08 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 8 09:13:56 2016 -0700

--
 .../test/IntegrationTestBigLinkedList.java  | 397 ++-
 ...egrationTestBigLinkedListWithVisibility.java |  15 +-
 .../hbase/test/IntegrationTestReplication.java  |   3 +-
 3 files changed, 318 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0d8e7b2f/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
--
diff --git 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 7f0f732..1f57e07 100644
--- 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -169,8 +169,9 @@ import com.google.common.collect.Sets;
  *
  * Below is a description of the Java programs
  *
- * Generator - A map only job that generates data. As stated previously,·
- * its best to generate data in multiples of 25M.
+ * Generator - A map only job that generates data. As stated previously,·its 
best to generate data
+ * in multiples of 25M. An option is also available to allow concurrent 
walkers to select and walk
+ * random flushed loops during this phase.
  *
  * Verify - A map reduce job that looks for holes. Look at the counts after 
running. REFERENCED and
  * UNREFERENCED are· ok, any UNDEFINED counts are bad. Do not run at the· 
same
@@ -183,6 +184,11 @@ import com.google.common.collect.Sets;
  * Delete - A standalone program that deletes a single node
  *
  * This class can be run as a unit test, as an integration test, or from the 
command line
+ *
+ * ex:
+ * ./hbase org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList
+ *loop 2 1 10 /temp 1 1000 50 1 0
+ *
  */
 @Category(IntegrationTests.class)
 public class IntegrationTestBigLinkedList extends IntegrationTestBase {
@@ -218,6 +224,9 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   private static final String GENERATOR_WRAP_KEY
 = "IntegrationTestBigLinkedList.generator.wrap";
 
+  private static final String CONCURRENT_WALKER_KEY
+= "IntegrationTestBigLinkedList.generator.concurrentwalkers";
+
   protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster
 
   private static final int MISSING_ROWS_TO_LOG = 10; // YARN complains when 
too many counters
@@ -226,6 +235,8 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   private static final int WRAP_DEFAULT = 25;
   private static final int ROWKEY_LENGTH = 16;
 
+  private static final int CONCURRENT_WALKER_DEFAULT = 0;
+
   protected String toRun;
   protected String[] otherArgs;
 
@@ -257,6 +268,18 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
 public static final String MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY =
 "generator.multiple.columnfamilies";
 
+public static enum Counts {
+  SUCCESS, TERMINATING, UNDEFINED, IOEXCEPTION
+}
+
+public static final String USAGE =  "Usage : " + 
Generator.class.getSimpleName() +
+"[ " +
+" ] \n" +
+"where  should be a multiple of width*wrap 
multiplier, 25M by default \n" +
+"walkers will verify random flushed loop during Generation.";
+
+public Job job;
+
 static class GeneratorInputFormat extends 
InputFormat {
   static class GeneratorInputSplit extends InputSplit implements Writable {
 @Override
@@ -372,6 +395,7 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
  * |___|
  * 
  */
+
 static class GeneratorMapper
   extends Mapper {
 
@@ -389,6 +413,12 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   boolean multipleUnevenColumnFamilies;
   byte[] tinyValue = new byte[] { 't' };
   byte[] bigValue = 

hbase git commit: HBASE-15935 Set up a concurrent walker that walks flushed circular linked lists as a Loop Mode

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 17edca634 -> cb118c8de


HBASE-15935 Set up a concurrent walker that walks flushed circular linked lists 
as a Loop Mode

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb118c8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb118c8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb118c8d

Branch: refs/heads/master
Commit: cb118c8de6ddb783e90c07912a5fbdd629eabf06
Parents: 17edca6
Author: Joseph Hwang 
Authored: Thu Jun 2 18:00:08 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 8 09:09:02 2016 -0700

--
 .../test/IntegrationTestBigLinkedList.java  | 397 ++-
 ...egrationTestBigLinkedListWithVisibility.java |  15 +-
 .../hbase/test/IntegrationTestReplication.java  |   3 +-
 3 files changed, 318 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cb118c8d/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
--
diff --git 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 430c8a6..6c54dca 100644
--- 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -168,8 +168,9 @@ import com.google.common.collect.Sets;
  *
  * Below is a description of the Java programs
  *
- * Generator - A map only job that generates data. As stated previously,·
- * its best to generate data in multiples of 25M.
+ * Generator - A map only job that generates data. As stated previously,·its 
best to generate data
+ * in multiples of 25M. An option is also available to allow concurrent 
walkers to select and walk
+ * random flushed loops during this phase.
  *
  * Verify - A map reduce job that looks for holes. Look at the counts after 
running. REFERENCED and
  * UNREFERENCED are· ok, any UNDEFINED counts are bad. Do not run at the· 
same
@@ -182,6 +183,11 @@ import com.google.common.collect.Sets;
  * Delete - A standalone program that deletes a single node
  *
  * This class can be run as a unit test, as an integration test, or from the 
command line
+ *
+ * ex:
+ * ./hbase org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList
+ *loop 2 1 10 /temp 1 1000 50 1 0
+ *
  */
 @Category(IntegrationTests.class)
 public class IntegrationTestBigLinkedList extends IntegrationTestBase {
@@ -217,6 +223,9 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   private static final String GENERATOR_WRAP_KEY
 = "IntegrationTestBigLinkedList.generator.wrap";
 
+  private static final String CONCURRENT_WALKER_KEY
+= "IntegrationTestBigLinkedList.generator.concurrentwalkers";
+
   protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster
 
   private static final int MISSING_ROWS_TO_LOG = 10; // YARN complains when 
too many counters
@@ -225,6 +234,8 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   private static final int WRAP_DEFAULT = 25;
   private static final int ROWKEY_LENGTH = 16;
 
+  private static final int CONCURRENT_WALKER_DEFAULT = 0;
+
   protected String toRun;
   protected String[] otherArgs;
 
@@ -256,6 +267,18 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
 public static final String MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY =
 "generator.multiple.columnfamilies";
 
+public static enum Counts {
+  SUCCESS, TERMINATING, UNDEFINED, IOEXCEPTION
+}
+
+public static final String USAGE =  "Usage : " + 
Generator.class.getSimpleName() +
+"[ " +
+" ] \n" +
+"where  should be a multiple of width*wrap 
multiplier, 25M by default \n" +
+"walkers will verify random flushed loop during Generation.";
+
+public Job job;
+
 static class GeneratorInputFormat extends 
InputFormat {
   static class GeneratorInputSplit extends InputSplit implements Writable {
 @Override
@@ -371,6 +394,7 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
  * |___|
  * 
  */
+
 static class GeneratorMapper
   extends Mapper {
 
@@ -388,6 +412,12 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   boolean multipleUnevenColumnFamilies;
   byte[] tinyValue = new byte[] { 't' };
   byte[] bigValue = null;
+  

hbase git commit: HBASE-15935 Set up a concurrent walker that walks flushed circular linked lists as a Loop Mode

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 ee63706ee -> 5ad4a824f


HBASE-15935 Set up a concurrent walker that walks flushed circular linked lists 
as a Loop Mode

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ad4a824
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ad4a824
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ad4a824

Branch: refs/heads/branch-1.3
Commit: 5ad4a824f1b39df7e6cdd22cac8146c84032496a
Parents: ee63706
Author: Joseph Hwang 
Authored: Thu Jun 2 18:00:08 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jul 8 09:14:08 2016 -0700

--
 .../test/IntegrationTestBigLinkedList.java  | 397 ++-
 ...egrationTestBigLinkedListWithVisibility.java |  15 +-
 .../hbase/test/IntegrationTestReplication.java  |   3 +-
 3 files changed, 318 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5ad4a824/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
--
diff --git 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 7f0f732..1f57e07 100644
--- 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -169,8 +169,9 @@ import com.google.common.collect.Sets;
  *
  * Below is a description of the Java programs
  *
- * Generator - A map only job that generates data. As stated previously,·
- * its best to generate data in multiples of 25M.
+ * Generator - A map only job that generates data. As stated previously,·its 
best to generate data
+ * in multiples of 25M. An option is also available to allow concurrent 
walkers to select and walk
+ * random flushed loops during this phase.
  *
  * Verify - A map reduce job that looks for holes. Look at the counts after 
running. REFERENCED and
  * UNREFERENCED are· ok, any UNDEFINED counts are bad. Do not run at the· 
same
@@ -183,6 +184,11 @@ import com.google.common.collect.Sets;
  * Delete - A standalone program that deletes a single node
  *
  * This class can be run as a unit test, as an integration test, or from the 
command line
+ *
+ * ex:
+ * ./hbase org.apache.hadoop.hbase.test.IntegrationTestBigLinkedList
+ *loop 2 1 10 /temp 1 1000 50 1 0
+ *
  */
 @Category(IntegrationTests.class)
 public class IntegrationTestBigLinkedList extends IntegrationTestBase {
@@ -218,6 +224,9 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   private static final String GENERATOR_WRAP_KEY
 = "IntegrationTestBigLinkedList.generator.wrap";
 
+  private static final String CONCURRENT_WALKER_KEY
+= "IntegrationTestBigLinkedList.generator.concurrentwalkers";
+
   protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster
 
   private static final int MISSING_ROWS_TO_LOG = 10; // YARN complains when 
too many counters
@@ -226,6 +235,8 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   private static final int WRAP_DEFAULT = 25;
   private static final int ROWKEY_LENGTH = 16;
 
+  private static final int CONCURRENT_WALKER_DEFAULT = 0;
+
   protected String toRun;
   protected String[] otherArgs;
 
@@ -257,6 +268,18 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
 public static final String MULTIPLE_UNEVEN_COLUMNFAMILIES_KEY =
 "generator.multiple.columnfamilies";
 
+public static enum Counts {
+  SUCCESS, TERMINATING, UNDEFINED, IOEXCEPTION
+}
+
+public static final String USAGE =  "Usage : " + 
Generator.class.getSimpleName() +
+"[ " +
+" ] \n" +
+"where  should be a multiple of width*wrap 
multiplier, 25M by default \n" +
+"walkers will verify random flushed loop during Generation.";
+
+public Job job;
+
 static class GeneratorInputFormat extends 
InputFormat {
   static class GeneratorInputSplit extends InputSplit implements Writable {
 @Override
@@ -372,6 +395,7 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
  * |___|
  * 
  */
+
 static class GeneratorMapper
   extends Mapper {
 
@@ -389,6 +413,12 @@ public class IntegrationTestBigLinkedList extends 
IntegrationTestBase {
   boolean multipleUnevenColumnFamilies;
   byte[] tinyValue = new byte[] { 't' };
   byte[] bigValue = 

hbase git commit: HBASE-16174 Hook cell test up, and fix broken cell test.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 ad276ef32 -> 114817253


HBASE-16174 Hook cell test up, and fix broken cell test.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/11481725
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/11481725
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/11481725

Branch: refs/heads/HBASE-14850
Commit: 114817253178f4faef00db93a187e7076fac658d
Parents: ad276ef
Author: Elliott Clark 
Authored: Tue Jul 5 11:13:01 2016 -0700
Committer: Elliott Clark 
Committed: Wed Jul 6 12:11:09 2016 -0700

--
 hbase-native-client/core/BUCK | 8 
 hbase-native-client/core/cell-test.cc | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/11481725/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 1c926e3..c615426 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -20,6 +20,7 @@ cxx_library(
 name="core",
 exported_headers=[
 "client.h",
+"cell.h",
 "hbase_macros.h",
 "region-location.h",
 "location-cache.h",
@@ -28,6 +29,7 @@ cxx_library(
 "meta-utils.h",
 ],
 srcs=[
+"cell.cc",
 "client.cc",
 "location-cache.cc",
 "meta-utils.cc",
@@ -51,6 +53,12 @@ cxx_test(name="location-cache-test",
  deps=[":core",
"//test-util:test-util", ],
  run_test_separately=True, )
+cxx_test(name="cell-test",
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
 cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/11481725/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
index cbe50eb..49f567b 100644
--- a/hbase-native-client/core/cell-test.cc
+++ b/hbase-native-client/core/cell-test.cc
@@ -100,7 +100,7 @@ TEST(CellTest, MultipleCellsTest) {
 
 TEST(CellTest, CellRowTest) {
   std::string row = "only-row";
-  std::string family = "";
+  std::string family = "D";
   std::string column = "";
   std::string value = "";
   long timestamp = std::numeric_limits::max();

hbase git commit: HBASE-16087 Replication shouldn't start on a master if if only hosts system tables

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 20a99b4c0 -> ae92668dd


HBASE-16087 Replication shouldn't start on a master if if only hosts system 
tables


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ae92668d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ae92668d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ae92668d

Branch: refs/heads/master
Commit: ae92668dd6eff5271ceeecc435165f5fc14fab48
Parents: 20a99b4
Author: Elliott Clark 
Authored: Wed Jun 22 15:40:28 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 5 13:50:37 2016 -0700

--
 .../hadoop/hbase/master/balancer/BaseLoadBalancer.java | 13 +
 .../hadoop/hbase/regionserver/HRegionServer.java   |  6 ++
 2 files changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ae92668d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index f52dbdf..dc5bace 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -1016,6 +1016,19 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 return tables != null && tables.length > 0;
   }
 
+  public static boolean userTablesOnMaster(Configuration conf) {
+String[] tables = getTablesOnMaster(conf);
+if (tables == null || tables.length == 0) {
+  return false;
+}
+for (String tn:tables) {
+  if (!tn.startsWith("hbase:")) {
+return true;
+  }
+}
+return false;
+  }
+
   @Override
   public void setConf(Configuration conf) {
 setSlop(conf);

http://git-wip-us.apache.org/repos/asf/hbase/blob/ae92668d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 1c1000e..877c3db 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -118,6 +118,7 @@ import org.apache.hadoop.hbase.ipc.ServerRpcController;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.RegionState.State;
 import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
 import org.apache.hadoop.hbase.mob.MobCacheConfig;
 import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -2651,6 +2652,11 @@ public class HRegionServer extends HasThread implements
   static private void createNewReplicationInstance(Configuration conf,
 HRegionServer server, FileSystem fs, Path logDir, Path oldLogDir) throws 
IOException{
 
+if ((server instanceof HMaster) &&
+(!BaseLoadBalancer.userTablesOnMaster(conf))) {
+  return;
+}
+
 // read in the name of the source replication class from the config file.
 String sourceClassname = 
conf.get(HConstants.REPLICATION_SOURCE_SERVICE_CLASSNAME,

HConstants.REPLICATION_SERVICE_CLASSNAME_DEFAULT);

hbase git commit: HBASE-16087 Replication shouldn't start on a master if if only hosts system tables

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 603decdbf -> 59c5900fa


HBASE-16087 Replication shouldn't start on a master if if only hosts system 
tables


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59c5900f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59c5900f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59c5900f

Branch: refs/heads/branch-1.3
Commit: 59c5900fae4392b9a5fcca8dbf5543e1bea1e452
Parents: 603decd
Author: Elliott Clark 
Authored: Wed Jun 22 15:40:28 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 5 13:48:28 2016 -0700

--
 .../hbase/master/balancer/BaseLoadBalancer.java   | 13 +
 .../hadoop/hbase/regionserver/HRegionServer.java  | 18 --
 2 files changed, 25 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/59c5900f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index b446172..c2529a8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -1001,6 +1001,19 @@ public abstract class BaseLoadBalancer implements 
LoadBalancer {
 return tables != null && tables.length > 0;
   }
 
+  public static boolean userTablesOnMaster(Configuration conf) {
+String[] tables = getTablesOnMaster(conf);
+if (tables == null || tables.length == 0) {
+  return false;
+}
+for (String tn:tables) {
+  if (!tn.startsWith("hbase:")) {
+return true;
+  }
+}
+return false;
+  }
+
   @Override
   public void setConf(Configuration conf) {
 setSlop(conf);

http://git-wip-us.apache.org/repos/asf/hbase/blob/59c5900f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 2fe9200..0c6ee83 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -106,6 +106,7 @@ import org.apache.hadoop.hbase.ipc.ServerRpcController;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.master.RegionState.State;
 import org.apache.hadoop.hbase.master.TableLockManager;
+import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
 import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.RequestConverter;
@@ -446,7 +447,7 @@ public class HRegionServer extends HasThread implements
   private RegionServerCoprocessorHost rsHost;
 
   private RegionServerProcedureManagerHost rspmHost;
-  
+
   private RegionServerQuotaManager rsQuotaManager;
 
   // Table level lock manager for locking for region operations
@@ -874,7 +875,7 @@ public class HRegionServer extends HasThread implements
 
 // Setup the Quota Manager
 rsQuotaManager = new RegionServerQuotaManager(this);
-
+
 // Setup RPC client for master communication
 rpcClient = RpcClientFactory.createClient(conf, clusterId, new 
InetSocketAddress(
 rpcServices.isa.getAddress(), 0), 
clusterConnection.getConnectionMetrics());
@@ -943,7 +944,7 @@ public class HRegionServer extends HasThread implements
 // since the server is ready to run
 rspmHost.start();
   }
-  
+
   // Start the Quota Manager
   if (this.rsQuotaManager != null) {
 rsQuotaManager.start(getRpcServer().getScheduler());
@@ -1037,7 +1038,7 @@ public class HRegionServer extends HasThread implements
 if (rsQuotaManager != null) {
   rsQuotaManager.stop();
 }
-
+
 // Stop the snapshot and other procedure handlers, forcefully killing all 
running tasks
 if (rspmHost != null) {
   rspmHost.stop(this.abortRequested || this.killed);
@@ -2598,7 +2599,7 @@ public class HRegionServer extends HasThread implements
   public ChoreService getChoreService() {
 return choreService;
   }
-  
+
   @Override
   public RegionServerQuotaManager getRegionServerQuotaManager() {
 return rsQuotaManager;
@@ -2620,6 +2621,11 @@ public class HRegionServer extends 

hbase git commit: HBASE-16177 In dev mode thrift server can't be run

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 8efc6148b -> 1318e84e1


HBASE-16177 In dev mode thrift server can't be run


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1318e84e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1318e84e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1318e84e

Branch: refs/heads/branch-1
Commit: 1318e84e14112a524935f50a380b9a9da29385fd
Parents: 8efc614
Author: Elliott Clark 
Authored: Tue Jul 5 11:55:53 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 5 12:15:31 2016 -0700

--
 hbase-assembly/pom.xml | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1318e84e/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index f79ba6b..1c256bb 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -162,6 +162,10 @@
   hbase-server
 
 
+  org.apache.hbase
+  hbase-thrift
+
+
 org.apache.hbase
 hbase-hadoop-compat
 

hbase git commit: HBASE-16177 In dev mode thrift server can't be run

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 318751cfd -> 2eef33930


HBASE-16177 In dev mode thrift server can't be run


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2eef3393
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2eef3393
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2eef3393

Branch: refs/heads/master
Commit: 2eef33930c358fa00347376604c3fc4ee68019c1
Parents: 318751c
Author: Elliott Clark 
Authored: Tue Jul 5 11:55:53 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 5 12:15:21 2016 -0700

--
 hbase-assembly/pom.xml | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2eef3393/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index c21d174..04c3981 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -162,6 +162,10 @@
   hbase-server
 
 
+  org.apache.hbase
+  hbase-thrift
+
+
 org.apache.hbase
 hbase-hadoop-compat
 

hbase git commit: HBASE-16177 In dev mode thrift server can't be run

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 b3834d7f7 -> 603decdbf


HBASE-16177 In dev mode thrift server can't be run


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/603decdb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/603decdb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/603decdb

Branch: refs/heads/branch-1.3
Commit: 603decdbf7eea4f86386496d141d3548f384f409
Parents: b3834d7
Author: Elliott Clark 
Authored: Tue Jul 5 11:55:53 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jul 5 12:15:41 2016 -0700

--
 hbase-assembly/pom.xml | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/603decdb/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 550a7c8..eca3f12 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -162,6 +162,10 @@
   hbase-server
 
 
+  org.apache.hbase
+  hbase-thrift
+
+
 org.apache.hbase
 hbase-hadoop-compat
 

hbase git commit: HBASE-15945 Patch for Cell

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 6c0216034 -> ad276ef32


HBASE-15945 Patch for Cell

This patch consists of Cell implementation without additional interfaces.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ad276ef3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ad276ef3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ad276ef3

Branch: refs/heads/HBASE-14850
Commit: ad276ef3225ee2058dc2fec7b866b3eaa4d537cb
Parents: 6c02160
Author: Sudeep Sunthankar 
Authored: Mon Jul 4 21:02:25 2016 +1000
Committer: Elliott Clark 
Committed: Tue Jul 5 10:43:41 2016 -0700

--
 hbase-native-client/core/cell-test.cc | 175 +
 hbase-native-client/core/cell.cc  |  77 +
 hbase-native-client/core/cell.h   |  60 ++
 3 files changed, 312 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ad276ef3/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
new file mode 100644
index 000..cbe50eb
--- /dev/null
+++ b/hbase-native-client/core/cell-test.cc
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+
+#include 
+#include 
+#include 
+
+using namespace hbase;
+TEST(CellTest, CellFailureTest) {
+  CellType cell_type = CellType::PUT;
+  std::string row = "row";
+  std::string family = "family";
+  std::string column = "column";
+  std::string value = "value";
+  long timestamp = std::numeric_limits::max();
+  std::string tags = "";
+  std::unique_ptr cell(
+  new Cell(row, family, column, timestamp, value, cell_type));
+  if (cell.get()) {
+EXPECT_NE("row-value", cell.get()->Row());
+EXPECT_NE("family-value", cell.get()->Family());
+EXPECT_NE("column-value", cell.get()->Qualifier());
+EXPECT_NE("value-value", cell.get()->Value());
+EXPECT_NE(8975431260, cell.get()->Timestamp());
+EXPECT_NE(CellType::MAXIMUM, cell.get()->Type());
+  }
+}
+
+TEST(CellTest, CellSuceessTest) {
+  std::string row = "row-value";
+  std::string family = "family-value";
+  std::string column = "column-value";
+  std::string value = "value-value";
+  long timestamp = std::numeric_limits::max();
+  CellType cell_type = CellType::PUT;
+  const std::unique_ptr cell(
+  new Cell(row, family, column, timestamp, value, cell_type));
+  if (cell.get()) {
+EXPECT_EQ(row, cell.get()->Row());
+EXPECT_EQ(family, cell.get()->Family());
+EXPECT_EQ(column, cell.get()->Qualifier());
+EXPECT_EQ(value, cell.get()->Value());
+EXPECT_EQ(timestamp, cell.get()->Timestamp());
+EXPECT_EQ(cell_type, cell.get()->Type());
+  }
+}
+
+TEST(CellTest, MultipleCellsTest) {
+  std::vector cells;
+  for (int i = 0; i < 5; i++) {
+std::string row = "row-value";
+std::string family = "family-value";
+std::string column = "column-value";
+std::string value = "value-value";
+long timestamp = std::numeric_limits::max();
+row += std::to_string(i);
+value += std::to_string(i);
+CellType cell_type = CellType::PUT;
+const Cell *cell = new Cell(row, family, column, timestamp, value,
+cell_type);
+cells.push_back(cell);
+  }
+  int i = 0;
+  for (const auto cell : cells) {
+std::string row = "row-value";
+std::string value = "value-value";
+row += std::to_string(i);
+value += std::to_string(i);
+EXPECT_EQ(row, cell->Row());
+EXPECT_EQ("family-value", cell->Family());
+EXPECT_EQ("column-value", cell->Qualifier());
+EXPECT_EQ(value, cell->Value());
+EXPECT_EQ(std::numeric_limits::max(), cell->Timestamp());
+EXPECT_EQ(CellType::PUT, cell->Type());
+i += 1;
+  }
+  for (const auto cell : cells) {
+delete cell;
+  }
+  cells.clear();
+}
+
+TEST(CellTest, CellRowTest) {

hbase git commit: HBASE-16089 Add on FastPath for CoDel

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 8ef87ce43 -> f4c1fd1a4


HBASE-16089 Add on FastPath for CoDel


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f4c1fd1a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f4c1fd1a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f4c1fd1a

Branch: refs/heads/branch-1
Commit: f4c1fd1a478631b702320a61e78c4647e4fea449
Parents: 8ef87ce
Author: Elliott Clark 
Authored: Wed Jun 22 16:34:40 2016 -0700
Committer: Elliott Clark 
Committed: Sat Jun 25 12:59:18 2016 -0700

--
 .../hbase/ipc/AdaptiveLifoCoDelCallQueue.java   |  59 ++---
 .../org/apache/hadoop/hbase/ipc/CallRunner.java |   4 +-
 .../ipc/FastPathBalancedQueueRpcExecutor.java   | 126 +++
 ...ifoWithFastPathBalancedQueueRpcExecutor.java | 116 -
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java|  12 +-
 .../hbase/ipc/TestSimpleRpcScheduler.java   |  67 ++
 6 files changed, 221 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f4c1fd1a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
index 08c488b..42b500f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
@@ -73,7 +73,7 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
   private AtomicBoolean resetDelay = new AtomicBoolean(true);
 
   // if we're in this mode, "long" calls are getting dropped
-  private volatile boolean isOverloaded;
+  private AtomicBoolean isOverloaded = new AtomicBoolean(false);
 
   public AdaptiveLifoCoDelCallQueue(int capacity, int targetDelay, int 
interval,
   double lifoThreshold, AtomicLong numGeneralCallsDropped, AtomicLong 
numLifoModeSwitches) {
@@ -126,6 +126,34 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
 }
   }
 
+  @Override
+  public CallRunner poll() {
+CallRunner cr;
+boolean switched = false;
+while(true) {
+  if (((double) queue.size() / this.maxCapacity) > lifoThreshold) {
+// Only count once per switch.
+if (!switched) {
+  switched = true;
+  numLifoModeSwitches.incrementAndGet();
+}
+cr = queue.pollLast();
+  } else {
+switched = false;
+cr = queue.pollFirst();
+  }
+  if (cr == null) {
+return cr;
+  }
+  if (needToDrop(cr)) {
+numGeneralCallsDropped.incrementAndGet();
+cr.drop();
+  } else {
+return cr;
+  }
+}
+  }
+
   /**
* @param callRunner to validate
* @return true if this call needs to be skipped based on call timestamp
@@ -136,28 +164,28 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
 long callDelay = now - callRunner.getCall().timestamp;
 
 long localMinDelay = this.minDelay;
-if (now > intervalTime && !resetDelay.getAndSet(true)) {
+
+// Try and determine if we should reset
+// the delay time and determine overload
+if (now > intervalTime &&
+!resetDelay.get() &&
+!resetDelay.getAndSet(true)) {
   intervalTime = now + codelInterval;
 
-  if (localMinDelay > codelTargetDelay) {
-isOverloaded = true;
-  } else {
-isOverloaded = false;
-  }
+  isOverloaded.set(localMinDelay > codelTargetDelay);
 }
 
-if (resetDelay.getAndSet(false)) {
+// If it looks like we should reset the delay
+// time do it only once on one thread
+if (resetDelay.get() && resetDelay.getAndSet(false)) {
   minDelay = callDelay;
+  // we just reset the delay dunno about how this will work
   return false;
 } else if (callDelay < localMinDelay) {
   minDelay = callDelay;
 }
 
-if (isOverloaded && callDelay > 2 * codelTargetDelay) {
-  return true;
-} else {
-  return false;
-}
+return isOverloaded.get() && callDelay > 2 * codelTargetDelay;
   }
 
   // Generic BlockingQueue methods we support
@@ -185,11 +213,6 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
   + " but take() and offer() methods");
   }
 
-  @Override
-  public CallRunner poll() {
-throw new UnsupportedOperationException("This class doesn't support 
anything,"
-  + " but take() and offer() methods");
-  }
 
   @Override
   public CallRunner peek() {

hbase git commit: HBASE-16089 Add on FastPath for CoDel

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 1d06850f4 -> fc4b8aa89


HBASE-16089 Add on FastPath for CoDel


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fc4b8aa8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fc4b8aa8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fc4b8aa8

Branch: refs/heads/master
Commit: fc4b8aa89d22961feb1ab7849104e68f6e1c0153
Parents: 1d06850
Author: Elliott Clark 
Authored: Wed Jun 22 16:34:40 2016 -0700
Committer: Elliott Clark 
Committed: Sat Jun 25 12:57:15 2016 -0700

--
 .../hbase/ipc/AdaptiveLifoCoDelCallQueue.java   |  59 ++---
 .../org/apache/hadoop/hbase/ipc/CallRunner.java |   4 +-
 .../ipc/FastPathBalancedQueueRpcExecutor.java   | 126 +++
 ...ifoWithFastPathBalancedQueueRpcExecutor.java | 116 -
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java|  14 +--
 .../hbase/ipc/TestSimpleRpcScheduler.java   |  74 +++
 6 files changed, 227 insertions(+), 166 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fc4b8aa8/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
index 08c488b..42b500f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
@@ -73,7 +73,7 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
   private AtomicBoolean resetDelay = new AtomicBoolean(true);
 
   // if we're in this mode, "long" calls are getting dropped
-  private volatile boolean isOverloaded;
+  private AtomicBoolean isOverloaded = new AtomicBoolean(false);
 
   public AdaptiveLifoCoDelCallQueue(int capacity, int targetDelay, int 
interval,
   double lifoThreshold, AtomicLong numGeneralCallsDropped, AtomicLong 
numLifoModeSwitches) {
@@ -126,6 +126,34 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
 }
   }
 
+  @Override
+  public CallRunner poll() {
+CallRunner cr;
+boolean switched = false;
+while(true) {
+  if (((double) queue.size() / this.maxCapacity) > lifoThreshold) {
+// Only count once per switch.
+if (!switched) {
+  switched = true;
+  numLifoModeSwitches.incrementAndGet();
+}
+cr = queue.pollLast();
+  } else {
+switched = false;
+cr = queue.pollFirst();
+  }
+  if (cr == null) {
+return cr;
+  }
+  if (needToDrop(cr)) {
+numGeneralCallsDropped.incrementAndGet();
+cr.drop();
+  } else {
+return cr;
+  }
+}
+  }
+
   /**
* @param callRunner to validate
* @return true if this call needs to be skipped based on call timestamp
@@ -136,28 +164,28 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
 long callDelay = now - callRunner.getCall().timestamp;
 
 long localMinDelay = this.minDelay;
-if (now > intervalTime && !resetDelay.getAndSet(true)) {
+
+// Try and determine if we should reset
+// the delay time and determine overload
+if (now > intervalTime &&
+!resetDelay.get() &&
+!resetDelay.getAndSet(true)) {
   intervalTime = now + codelInterval;
 
-  if (localMinDelay > codelTargetDelay) {
-isOverloaded = true;
-  } else {
-isOverloaded = false;
-  }
+  isOverloaded.set(localMinDelay > codelTargetDelay);
 }
 
-if (resetDelay.getAndSet(false)) {
+// If it looks like we should reset the delay
+// time do it only once on one thread
+if (resetDelay.get() && resetDelay.getAndSet(false)) {
   minDelay = callDelay;
+  // we just reset the delay dunno about how this will work
   return false;
 } else if (callDelay < localMinDelay) {
   minDelay = callDelay;
 }
 
-if (isOverloaded && callDelay > 2 * codelTargetDelay) {
-  return true;
-} else {
-  return false;
-}
+return isOverloaded.get() && callDelay > 2 * codelTargetDelay;
   }
 
   // Generic BlockingQueue methods we support
@@ -185,11 +213,6 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
   + " but take() and offer() methods");
   }
 
-  @Override
-  public CallRunner poll() {
-throw new UnsupportedOperationException("This class doesn't support 
anything,"
-  + " but take() and offer() methods");
-  }
 
   @Override
   public CallRunner peek() {

hbase git commit: HBASE-16089 Add on FastPath for CoDel

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 e7bfd9bfe -> 65cbb6c2e


HBASE-16089 Add on FastPath for CoDel


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/65cbb6c2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/65cbb6c2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/65cbb6c2

Branch: refs/heads/branch-1.3
Commit: 65cbb6c2e7335596c6edf8c0243bcd66722641bb
Parents: e7bfd9b
Author: Elliott Clark 
Authored: Wed Jun 22 16:34:40 2016 -0700
Committer: Elliott Clark 
Committed: Sat Jun 25 12:59:37 2016 -0700

--
 .../hbase/ipc/AdaptiveLifoCoDelCallQueue.java   |  59 ++---
 .../org/apache/hadoop/hbase/ipc/CallRunner.java |   4 +-
 .../ipc/FastPathBalancedQueueRpcExecutor.java   | 126 +++
 ...ifoWithFastPathBalancedQueueRpcExecutor.java | 116 -
 .../hadoop/hbase/ipc/SimpleRpcScheduler.java|  12 +-
 .../hbase/ipc/TestSimpleRpcScheduler.java   |  67 ++
 6 files changed, 221 insertions(+), 163 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/65cbb6c2/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
index 08c488b..42b500f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/AdaptiveLifoCoDelCallQueue.java
@@ -73,7 +73,7 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
   private AtomicBoolean resetDelay = new AtomicBoolean(true);
 
   // if we're in this mode, "long" calls are getting dropped
-  private volatile boolean isOverloaded;
+  private AtomicBoolean isOverloaded = new AtomicBoolean(false);
 
   public AdaptiveLifoCoDelCallQueue(int capacity, int targetDelay, int 
interval,
   double lifoThreshold, AtomicLong numGeneralCallsDropped, AtomicLong 
numLifoModeSwitches) {
@@ -126,6 +126,34 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
 }
   }
 
+  @Override
+  public CallRunner poll() {
+CallRunner cr;
+boolean switched = false;
+while(true) {
+  if (((double) queue.size() / this.maxCapacity) > lifoThreshold) {
+// Only count once per switch.
+if (!switched) {
+  switched = true;
+  numLifoModeSwitches.incrementAndGet();
+}
+cr = queue.pollLast();
+  } else {
+switched = false;
+cr = queue.pollFirst();
+  }
+  if (cr == null) {
+return cr;
+  }
+  if (needToDrop(cr)) {
+numGeneralCallsDropped.incrementAndGet();
+cr.drop();
+  } else {
+return cr;
+  }
+}
+  }
+
   /**
* @param callRunner to validate
* @return true if this call needs to be skipped based on call timestamp
@@ -136,28 +164,28 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
 long callDelay = now - callRunner.getCall().timestamp;
 
 long localMinDelay = this.minDelay;
-if (now > intervalTime && !resetDelay.getAndSet(true)) {
+
+// Try and determine if we should reset
+// the delay time and determine overload
+if (now > intervalTime &&
+!resetDelay.get() &&
+!resetDelay.getAndSet(true)) {
   intervalTime = now + codelInterval;
 
-  if (localMinDelay > codelTargetDelay) {
-isOverloaded = true;
-  } else {
-isOverloaded = false;
-  }
+  isOverloaded.set(localMinDelay > codelTargetDelay);
 }
 
-if (resetDelay.getAndSet(false)) {
+// If it looks like we should reset the delay
+// time do it only once on one thread
+if (resetDelay.get() && resetDelay.getAndSet(false)) {
   minDelay = callDelay;
+  // we just reset the delay dunno about how this will work
   return false;
 } else if (callDelay < localMinDelay) {
   minDelay = callDelay;
 }
 
-if (isOverloaded && callDelay > 2 * codelTargetDelay) {
-  return true;
-} else {
-  return false;
-}
+return isOverloaded.get() && callDelay > 2 * codelTargetDelay;
   }
 
   // Generic BlockingQueue methods we support
@@ -185,11 +213,6 @@ public class AdaptiveLifoCoDelCallQueue implements 
BlockingQueue {
   + " but take() and offer() methods");
   }
 
-  @Override
-  public CallRunner poll() {
-throw new UnsupportedOperationException("This class doesn't support 
anything,"
-  + " but take() and offer() methods");
-  }
 
   @Override
   public CallRunner peek() {

hbase git commit: HBASE-16093 Fix splits failed before creating daughter regions leave meta inconsistent

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 b6e2bc5ad -> 93d82112f


HBASE-16093 Fix splits failed before creating daughter regions leave meta 
inconsistent


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/93d82112
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/93d82112
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/93d82112

Branch: refs/heads/branch-1
Commit: 93d82112f00d44c3ded623354f16d3c062cf51e7
Parents: b6e2bc5
Author: Elliott Clark 
Authored: Thu Jun 23 11:27:44 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 23 12:21:52 2016 -0700

--
 .../hadoop/hbase/master/AssignmentManager.java   | 15 +--
 1 file changed, 13 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/93d82112/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 3130974..2efc77d 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -3758,9 +3758,20 @@ public class AssignmentManager extends ZooKeeperListener 
{
 if (!org.apache.commons.lang.StringUtils.isEmpty(s)) {
   return s;
 }
+
+// Always bring the parent back online. Even if it's not offline
+// There's no harm in making it online again.
 regionOnline(p, sn);
-regionOffline(a);
-regionOffline(b);
+
+// Only offline the region if they are known to exist.
+RegionState regionStateA = regionStates.getRegionState(a);
+RegionState regionStateB = regionStates.getRegionState(b);
+if (regionStateA != null) {
+  regionOffline(a);
+}
+if (regionStateB != null) {
+  regionOffline(b);
+}
 
 if (getTableStateManager().isTableState(p.getTable(),
 ZooKeeperProtos.Table.State.DISABLED, 
ZooKeeperProtos.Table.State.DISABLING)) {

hbase git commit: HBASE-16093 Fix splits failed before creating daughter regions leave meta inconsistent

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 d500f6fd0 -> 25485d621


HBASE-16093 Fix splits failed before creating daughter regions leave meta 
inconsistent


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/25485d62
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/25485d62
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/25485d62

Branch: refs/heads/branch-1.3
Commit: 25485d621c220e92b0248f3d57a3b3c529a86e47
Parents: d500f6f
Author: Elliott Clark 
Authored: Thu Jun 23 11:27:44 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 23 12:22:02 2016 -0700

--
 .../hadoop/hbase/master/AssignmentManager.java   | 15 +--
 1 file changed, 13 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/25485d62/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index b325f42..739ebff 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -3758,9 +3758,20 @@ public class AssignmentManager extends ZooKeeperListener 
{
 if (!org.apache.commons.lang.StringUtils.isEmpty(s)) {
   return s;
 }
+
+// Always bring the parent back online. Even if it's not offline
+// There's no harm in making it online again.
 regionOnline(p, sn);
-regionOffline(a);
-regionOffline(b);
+
+// Only offline the region if they are known to exist.
+RegionState regionStateA = regionStates.getRegionState(a);
+RegionState regionStateB = regionStates.getRegionState(b);
+if (regionStateA != null) {
+  regionOffline(a);
+}
+if (regionStateB != null) {
+  regionOffline(b);
+}
 
 if (getTableStateManager().isTableState(p.getTable(),
 ZooKeeperProtos.Table.State.DISABLED, 
ZooKeeperProtos.Table.State.DISABLING)) {

[1/3] hbase git commit: HBASE-14871 Allow specifying the base branch for make_patch

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 857ec2f44 -> b6e2bc5ad


HBASE-14871 Allow specifying the base branch for make_patch


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/28d1706d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/28d1706d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/28d1706d

Branch: refs/heads/branch-1
Commit: 28d1706d964cf6ba5cbdfe55521b20cb7fdf6600
Parents: 857ec2f
Author: Elliott Clark 
Authored: Mon Nov 23 16:48:45 2015 -0800
Committer: Elliott Clark 
Committed: Thu Jun 23 11:29:20 2016 -0700

--
 dev-support/make_patch.sh | 35 ++-
 1 file changed, 22 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/28d1706d/dev-support/make_patch.sh
--
diff --git a/dev-support/make_patch.sh b/dev-support/make_patch.sh
index ec2bbb9..446c0a5 100755
--- a/dev-support/make_patch.sh
+++ b/dev-support/make_patch.sh
@@ -20,18 +20,22 @@
 # Make a patch for the current branch based on its tracking branch
 
 # Process args
-while getopts "ahd:" opt; do
+while getopts "ahd:b:" opt; do
 case "$opt" in
 a)  addendum='-addendum'
 ;;
 d)  
 patch_dir=$OPTARG
 ;;
+b)
+tracking_branch=$OPTARG
+;;
 *)
 echo -e "Usage: $0 [-h] [-a] [-d]  \n\
 Must be run from within the git branch to make the patch against.\n\
 -h - display these instructions.\n\
 -a - Add an 'addendum' prefix to the patch name.\n\
+-b - Specify the base branch to diff from. (defaults to the tracking 
branch or origin master)\n\
 -d - specify a patch directory (defaults to ~/patches/)"
 exit 0
 ;;
@@ -53,20 +57,25 @@ if [ "$git_dirty" -ne 0 ]; then
 exit 1
 fi
 
-# Determine the tracking branch
-git log -n 1 origin/$branch > /dev/null 2>&1
-status=$?
-if [ "$status" -eq 128 ]; then
-# Status 128 means there is no remote branch
-tracking_branch='origin/master'
-elif [ "$status" -eq 0 ]; then
-# Status 0 means there is a remote branch
-tracking_branch="origin/$branch"
-else
-echo "Unknown error: $?" >&2
-exit 1
+# Determine the tracking branch if needed.
+# If it was passed in from the command line
+# with -b then use dthat no matter what.
+if [ ! "$tracking_branch" ]; then
+  git log -n 1 origin/$branch > /dev/null 2>&1
+  status=$?
+  if [ "$status" -eq 128 ]; then
+  # Status 128 means there is no remote branch
+  tracking_branch='origin/master'
+  elif [ "$status" -eq 0 ]; then
+  # Status 0 means there is a remote branch
+  tracking_branch="origin/$branch"
+  else
+  echo "Unknown error: $?" >&2
+  exit 1
+  fi
 fi
 
+
 # Deal with invalid or missing $patch_dir
 if [ ! "$patch_dir" ]; then
 echo -e "Patch directory not specified. Falling back to ~/patches/."

[3/3] hbase git commit: HBASE-15725 make_patch.sh should add the branch name when -b is passed. -- addendum

[eclark]

HBASE-15725 make_patch.sh should add the branch name when -b is passed. -- 
addendum


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b6e2bc5a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b6e2bc5a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b6e2bc5a

Branch: refs/heads/branch-1
Commit: b6e2bc5ad127c68dd6c933eba273a02f0455c8c1
Parents: c7e5672
Author: Elliott Clark 
Authored: Fri May 13 16:38:02 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 23 11:29:36 2016 -0700

--
 dev-support/make_patch.sh | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b6e2bc5a/dev-support/make_patch.sh
--
diff --git a/dev-support/make_patch.sh b/dev-support/make_patch.sh
index 0aedf88..8179098 100755
--- a/dev-support/make_patch.sh
+++ b/dev-support/make_patch.sh
@@ -97,7 +97,7 @@ elif  [ "$status" -ge 1 ]; then
 # At least one patch already exists -- add a version prefix
 for i in {1..99}; do
 # Check to see the maximum version of patch that exists
-if [ ! -f "$patch_dir/$branch-v$i.patch" ]; then
+if [ ! -f "$patch_dir/$branch.v$i.patch" ]; then
 version=$i
 if [ -n "$addendum" ]; then
 # Don't increment the patch # if it is an addendum
@@ -108,10 +108,10 @@ elif  [ "$status" -ge 1 ]; then
 else
 # We are making an addendum to a different version of the 
patch
 let version=$version-1
-prefix="-v$version"
+prefix=".v$version"
 fi
 else
-prefix="-v$version"
+prefix=".v$version"
 fi
 break
 fi
@@ -122,7 +122,7 @@ fi
 tracking_suffix=""
 if [[ $tracking_branch != "origin/master" \
 &&  $tracking_branch != "master" ]]; then
-tracking_suffix="-${tracking_branch#origin/}"
+tracking_suffix=".${tracking_branch#origin/}"
 fi
 
 patch_name="$branch$prefix$addendum$tracking_suffix.patch"

[2/3] hbase git commit: HBASE-15725 make_patch.sh should add the branch name when -b is passed.

[eclark]

HBASE-15725 make_patch.sh should add the branch name when -b is passed.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c7e5672d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c7e5672d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c7e5672d

Branch: refs/heads/branch-1
Commit: c7e5672d4d3d6a44ac01c0a56e122a58994846de
Parents: 28d1706
Author: Elliott Clark 
Authored: Thu May 12 11:15:57 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 23 11:29:27 2016 -0700

--
 dev-support/make_patch.sh | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c7e5672d/dev-support/make_patch.sh
--
diff --git a/dev-support/make_patch.sh b/dev-support/make_patch.sh
index 446c0a5..0aedf88 100755
--- a/dev-support/make_patch.sh
+++ b/dev-support/make_patch.sh
@@ -24,7 +24,7 @@ while getopts "ahd:b:" opt; do
 case "$opt" in
 a)  addendum='-addendum'
 ;;
-d)  
+d)
 patch_dir=$OPTARG
 ;;
 b)
@@ -52,7 +52,7 @@ fi
 # Exit if git status is dirty
 git_dirty=$(git diff --shortstat 2> /dev/null | wc -l|awk {'print $1'})
 echo "git_dirty is $git_dirty"
-if [ "$git_dirty" -ne 0 ]; then  
+if [ "$git_dirty" -ne 0 ]; then
 echo "Git status is dirty. Commit locally first.">&2
 exit 1
 fi
@@ -117,8 +117,15 @@ elif  [ "$status" -ge 1 ]; then
 fi
 done
 fi
+# If this is against a tracking branch other than master
+# include it in the patch name
+tracking_suffix=""
+if [[ $tracking_branch != "origin/master" \
+&&  $tracking_branch != "master" ]]; then
+tracking_suffix="-${tracking_branch#origin/}"
+fi
 
-patch_name="$branch$prefix$addendum.patch"
+patch_name="$branch$prefix$addendum$tracking_suffix.patch"
 
 # Do we need to make a diff?
 git diff --quiet $tracking_branch
@@ -134,10 +141,10 @@ local_commits=$(git log $tracking_branch..$branch|grep 
'Author:'|wc -l|awk {'pri
 if [ "$local_commits" -gt 1 ]; then
 read -p "$local_commits commits exist only in your local branch. 
Interactive rebase?" yn
 case $yn in
-[Yy]* ) 
+[Yy]* )
 git rebase -i $tracking_branch
 ;;
-[Nn]* ) 
+[Nn]* )
   echo "Creating $patch_dir/$patch_name using git diff."
   git diff $tracking_branch > $patch_dir/$patch_name
   exit 0
@@ -147,5 +154,3 @@ fi
 
 echo "Creating patch $patch_dir/$patch_name using git format-patch"
 git format-patch --stdout $tracking_branch > $patch_dir/$patch_name
-
-

hbase git commit: HBASE-16083 Fix RepliationFactory config property names to match HBase naming conventions

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master f3b3a7c9b -> 2766e4a42


HBASE-16083 Fix RepliationFactory config property names to match HBase naming 
conventions

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2766e4a4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2766e4a4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2766e4a4

Branch: refs/heads/master
Commit: 2766e4a42ec8d953c92527e4425e27cf82018a7a
Parents: f3b3a7c
Author: Joseph Hwang 
Authored: Wed Jun 22 09:31:36 2016 -0700
Committer: Elliott Clark 
Committed: Wed Jun 22 12:49:45 2016 -0700

--
 .../org/apache/hadoop/hbase/replication/ReplicationFactory.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2766e4a4/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 38f9f30..b84641c 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -34,7 +34,7 @@ public class ReplicationFactory {
   public static ReplicationQueues 
getReplicationQueues(ReplicationQueuesArguments args)
   throws Exception {
 Class classToBuild = args.getConf().getClass("hbase.region.replica." +
-"replication.ReplicationQueuesType", ReplicationQueuesZKImpl.class);
+"replication.replicationQueues.class", ReplicationQueuesZKImpl.class);
 return (ReplicationQueues) 
ConstructorUtils.invokeConstructor(classToBuild, args);
   }
 
@@ -42,7 +42,7 @@ public class ReplicationFactory {
   ReplicationQueuesClientArguments args)
 throws Exception {
 Class classToBuild = args.getConf().getClass("hbase.region.replica." +
-  "replication.ReplicationQueuesClientType", 
ReplicationQueuesClientZKImpl.class);
+  "replication.replicationQueuesClient.class", 
ReplicationQueuesClientZKImpl.class);
 return (ReplicationQueuesClient) 
ConstructorUtils.invokeConstructor(classToBuild, args);
   }
 

hbase git commit: HBASE-16080 Fix flakey test TestTableBasedReplicationSourceManager.cleanupFailoverQueues().

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 6fc71c5d3 -> f3b3a7c9b


HBASE-16080 Fix flakey test 
TestTableBasedReplicationSourceManager.cleanupFailoverQueues().

Instead of running the primary test in a separate thread and hoping it finishes 
in time, just run the test in the primary thread.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f3b3a7c9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f3b3a7c9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f3b3a7c9

Branch: refs/heads/master
Commit: f3b3a7c9b5c5e28545fb47dc501d6bfc6a4e1cfe
Parents: 6fc71c5
Author: Joseph Hwang 
Authored: Wed Jun 22 08:55:41 2016 -0700
Committer: Elliott Clark 
Committed: Wed Jun 22 12:26:35 2016 -0700

--
 .../replication/regionserver/TestReplicationSourceManager.java| 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f3b3a7c9/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 4b278bb..f60982e 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -345,8 +345,7 @@ public abstract class TestReplicationSourceManager {
 NodeFailoverWorker w1 =
 manager.new NodeFailoverWorker(server.getServerName().getServerName(), 
rq1, rp1, new UUID(
 new Long(1), new Long(2)));
-w1.start();
-w1.join(1);
+w1.run();
 assertEquals(1, manager.getWalsByIdRecoveredQueues().size());
 String id = "1-" + server.getServerName().getServerName();
 assertEquals(files, 
manager.getWalsByIdRecoveredQueues().get(id).get(group));

hbase git commit: HBASE-16036 Made Replication Table creation non-blocking.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master b006e41a3 -> 152594560


HBASE-16036 Made Replication Table creation non-blocking.

All ReplicationTableBase method's that need to access the Replication Table 
will block until it is created though.
Also refactored ReplicationSourceManager so that abandoned queue adoption is 
run in the background too so that it does not block HRegionServer 
initialization.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/15259456
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/15259456
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/15259456

Branch: refs/heads/master
Commit: 152594560e29549642587b850320f5d66339b747
Parents: b006e41
Author: Joseph Hwang 
Authored: Wed Jun 15 14:35:56 2016 -0700
Committer: Elliott Clark 
Committed: Tue Jun 21 13:05:50 2016 -0700

--
 .../replication/ReplicationQueuesArguments.java |   4 +
 .../ReplicationQueuesClientArguments.java   |   5 +
 .../hbase/replication/ReplicationTableBase.java | 186 ++-
 .../TableBasedReplicationQueuesClientImpl.java  |   3 +-
 .../TableBasedReplicationQueuesImpl.java| 106 +--
 .../regionserver/ReplicationSourceManager.java  |  43 +++--
 .../TestReplicationStateHBaseImpl.java  |   2 +-
 .../replication/TestReplicationTableBase.java   | 109 +++
 .../TestReplicationSourceManager.java   | 160 
 .../TestReplicationSourceManagerZkImpl.java | 152 +++
 ...tTableBasedReplicationSourceManagerImpl.java |  60 ++
 11 files changed, 579 insertions(+), 251 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/15259456/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
index 4fdc4e7..12fc6a1 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
@@ -23,6 +23,10 @@ import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
+/**
+ * Wrapper around common arguments used to construct ReplicationQueues. Used 
to construct various
+ * ReplicationQueues Implementations with different constructor arguments by 
reflection.
+ */
 @InterfaceAudience.Private
 public class ReplicationQueuesArguments {
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/15259456/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
index 8a61993..834f831 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
@@ -23,6 +23,11 @@ import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 
+/**
+ * Wrapper around common arguments used to construct ReplicationQueuesClient. 
Used to construct
+ * various ReplicationQueuesClient Implementations with different constructor 
arguments by
+ * reflection.
+ */
 @InterfaceAudience.Private
 public class ReplicationQueuesClientArguments extends 
ReplicationQueuesArguments {
   public ReplicationQueuesClientArguments(Configuration conf, Abortable abort,

http://git-wip-us.apache.org/repos/asf/hbase/blob/15259456/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
index c1506cd..61bb041 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
@@ -18,12 +18,14 @@
 */
 package 

hbase git commit: HBASE-16061 Allow logging to a buffered console

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 411e3cdb6 -> e721aa1a8


HBASE-16061 Allow logging to a buffered console


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e721aa1a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e721aa1a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e721aa1a

Branch: refs/heads/branch-1
Commit: e721aa1a8ac2db2697d699ca553fad4b26b5e633
Parents: 411e3cd
Author: Elliott Clark 
Authored: Fri Jun 17 16:32:57 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 17 17:09:18 2016 -0700

--
 conf/log4j.properties   |  5 +-
 .../hadoop/hbase/AsyncConsoleAppender.java  | 48 
 2 files changed, 52 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e721aa1a/conf/log4j.properties
--
diff --git a/conf/log4j.properties b/conf/log4j.properties
index a15d575..e527c9b 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -80,13 +80,16 @@ 
log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
 
 #
 # console
-# Add "console" to rootlogger above if you want to use this 
+# Add "console" to rootlogger above if you want to use this
 #
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: 
%m%n
 
+log4j.appender.asyncconsole=org.apache.hadoop.hbase.AsyncConsoleAppender
+log4j.appender.asyncconsole.target=System.err
+
 # Custom Logging levels
 
 log4j.logger.org.apache.zookeeper=INFO

http://git-wip-us.apache.org/repos/asf/hbase/blob/e721aa1a/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
new file mode 100644
index 000..338265d
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.log4j.AsyncAppender;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
+
+/**
+ * Logger class that buffers before trying to log to the specified console.
+ */
+@InterfaceAudience.Private
+public class AsyncConsoleAppender extends AsyncAppender {
+  private final ConsoleAppender consoleAppender;
+
+  public AsyncConsoleAppender() {
+super();
+consoleAppender = new ConsoleAppender(new PatternLayout(
+"%d{ISO8601} %-5p [%t] %c{2}: %m%n"));
+this.addAppender(consoleAppender);
+  }
+
+  public void setTarget(String value) {
+consoleAppender.setTarget(value);
+  }
+
+  public void activateOptions() {
+consoleAppender.activateOptions();
+super.activateOptions();
+  }
+
+}

hbase git commit: HBASE-16061 Allow logging to a buffered console

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 81a9c1ac3 -> 65a8d7743


HBASE-16061 Allow logging to a buffered console


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/65a8d774
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/65a8d774
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/65a8d774

Branch: refs/heads/master
Commit: 65a8d77433cf72ce67e8b5e2efb35f2c2603c349
Parents: 81a9c1a
Author: Elliott Clark 
Authored: Fri Jun 17 16:32:57 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 17 17:06:12 2016 -0700

--
 conf/log4j.properties   |  5 +-
 .../hadoop/hbase/AsyncConsoleAppender.java  | 48 
 2 files changed, 52 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/65a8d774/conf/log4j.properties
--
diff --git a/conf/log4j.properties b/conf/log4j.properties
index d36a22e..74b13b1 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -80,13 +80,16 @@ 
log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
 
 #
 # console
-# Add "console" to rootlogger above if you want to use this 
+# Add "console" to rootlogger above if you want to use this
 #
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: 
%m%n
 
+log4j.appender.asyncconsole=org.apache.hadoop.hbase.AsyncConsoleAppender
+log4j.appender.asyncconsole.target=System.err
+
 # Custom Logging levels
 
 log4j.logger.org.apache.zookeeper=INFO

http://git-wip-us.apache.org/repos/asf/hbase/blob/65a8d774/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
new file mode 100644
index 000..338265d
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.log4j.AsyncAppender;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
+
+/**
+ * Logger class that buffers before trying to log to the specified console.
+ */
+@InterfaceAudience.Private
+public class AsyncConsoleAppender extends AsyncAppender {
+  private final ConsoleAppender consoleAppender;
+
+  public AsyncConsoleAppender() {
+super();
+consoleAppender = new ConsoleAppender(new PatternLayout(
+"%d{ISO8601} %-5p [%t] %c{2}: %m%n"));
+this.addAppender(consoleAppender);
+  }
+
+  public void setTarget(String value) {
+consoleAppender.setTarget(value);
+  }
+
+  public void activateOptions() {
+consoleAppender.activateOptions();
+super.activateOptions();
+  }
+
+}

hbase git commit: HBASE-16061 Allow logging to a buffered console

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 c327d9e38 -> 75fb789a7


HBASE-16061 Allow logging to a buffered console


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75fb789a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75fb789a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75fb789a

Branch: refs/heads/branch-1.3
Commit: 75fb789a797de016217b24f4e6e9e3ef0a868cb3
Parents: c327d9e
Author: Elliott Clark 
Authored: Fri Jun 17 16:32:57 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 17 17:06:32 2016 -0700

--
 conf/log4j.properties   |  5 +-
 .../hadoop/hbase/AsyncConsoleAppender.java  | 48 
 2 files changed, 52 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/75fb789a/conf/log4j.properties
--
diff --git a/conf/log4j.properties b/conf/log4j.properties
index a15d575..e527c9b 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -80,13 +80,16 @@ 
log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
 
 #
 # console
-# Add "console" to rootlogger above if you want to use this 
+# Add "console" to rootlogger above if you want to use this
 #
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: 
%m%n
 
+log4j.appender.asyncconsole=org.apache.hadoop.hbase.AsyncConsoleAppender
+log4j.appender.asyncconsole.target=System.err
+
 # Custom Logging levels
 
 log4j.logger.org.apache.zookeeper=INFO

http://git-wip-us.apache.org/repos/asf/hbase/blob/75fb789a/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
new file mode 100644
index 000..338265d
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.log4j.AsyncAppender;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
+
+/**
+ * Logger class that buffers before trying to log to the specified console.
+ */
+@InterfaceAudience.Private
+public class AsyncConsoleAppender extends AsyncAppender {
+  private final ConsoleAppender consoleAppender;
+
+  public AsyncConsoleAppender() {
+super();
+consoleAppender = new ConsoleAppender(new PatternLayout(
+"%d{ISO8601} %-5p [%t] %c{2}: %m%n"));
+this.addAppender(consoleAppender);
+  }
+
+  public void setTarget(String value) {
+consoleAppender.setTarget(value);
+  }
+
+  public void activateOptions() {
+consoleAppender.activateOptions();
+super.activateOptions();
+  }
+
+}

hbase git commit: HBASE-16018 Refactored the ReplicationPeers interface to clear up what some methods do and move away from a ZooKeeper-specific implementation.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 5147fb12a -> 61ff6ced5


HBASE-16018 Refactored the ReplicationPeers interface to clear up what some 
methods do and move away from a ZooKeeper-specific implementation.

Also added some documentation for undocumented methods.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/61ff6ced
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/61ff6ced
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/61ff6ced

Branch: refs/heads/master
Commit: 61ff6ced5bea3586129fb0844bbf64b122775b42
Parents: 5147fb1
Author: Joseph Hwang 
Authored: Tue Jun 14 08:58:49 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 17 13:04:21 2016 -0700

--
 .../client/replication/ReplicationAdmin.java|  8 ++--
 .../hbase/replication/ReplicationPeers.java | 41 +++-
 .../replication/ReplicationPeersZKImpl.java | 17 
 .../regionserver/ReplicationSource.java |  2 +-
 .../regionserver/ReplicationSourceManager.java  | 14 +++
 .../cleaner/TestReplicationHFileCleaner.java|  4 +-
 .../replication/TestReplicationStateBasic.java  | 40 +--
 .../TestReplicationStateHBaseImpl.java  |  6 +--
 .../TestReplicationTrackerZKImpl.java   | 18 -
 9 files changed, 85 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/61ff6ced/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index e0985bd..b04d317 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -183,7 +183,7 @@ public class ReplicationAdmin implements Closeable {
 if (tableCfs != null) {
   peerConfig.setTableCFsMap(tableCfs);
 }
-this.replicationPeers.addPeer(id, peerConfig);
+this.replicationPeers.registerPeer(id, peerConfig);
   }
 
   /**
@@ -192,7 +192,7 @@ public class ReplicationAdmin implements Closeable {
* @param peerConfig configuration for the replication slave cluster
*/
   public void addPeer(String id, ReplicationPeerConfig peerConfig) throws 
ReplicationException {
-this.replicationPeers.addPeer(id, peerConfig);
+this.replicationPeers.registerPeer(id, peerConfig);
   }
 
   /**
@@ -212,7 +212,7 @@ public class ReplicationAdmin implements Closeable {
* @param id a short name that identifies the cluster
*/
   public void removePeer(String id) throws ReplicationException {
-this.replicationPeers.removePeer(id);
+this.replicationPeers.unregisterPeer(id);
   }
 
   /**
@@ -556,7 +556,7 @@ public class ReplicationAdmin implements Closeable {
 
   @VisibleForTesting
   public void peerAdded(String id) throws ReplicationException {
-this.replicationPeers.peerAdded(id);
+this.replicationPeers.peerConnected(id);
   }
 
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/hbase/blob/61ff6ced/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
index 9f70d95..2a7963a 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -51,18 +51,30 @@ public interface ReplicationPeers {
* @param peerId a short that identifies the cluster
* @param peerConfig configuration for the replication slave cluster
*/
-  void addPeer(String peerId, ReplicationPeerConfig peerConfig)
+  void registerPeer(String peerId, ReplicationPeerConfig peerConfig)
   throws ReplicationException;
 
   /**
* Removes a remote slave cluster and stops the replication to it.
* @param peerId a short that identifies the cluster
*/
-  void removePeer(String peerId) throws ReplicationException;
+  void unregisterPeer(String peerId) throws ReplicationException;
 
-  boolean peerAdded(String peerId) throws ReplicationException;
+  /**
+   * Method called after a peer has been connected. It will create a 
ReplicationPeer to track the
+   * newly connected cluster.
+   * @param peerId a short that 

hbase git commit: HBASE-15429 Add split policy for busy regions

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 6d0e0e372 -> 3abd52bdc


HBASE-15429 Add split policy for busy regions

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3abd52bd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3abd52bd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3abd52bd

Branch: refs/heads/master
Commit: 3abd52bdc6db926d930fd94cfce5bd7ba6fd005f
Parents: 6d0e0e3
Author: Ashu Pachauri 
Authored: Mon Apr 11 07:43:06 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 16 17:36:54 2016 -0700

--
 .../src/main/resources/hbase-default.xml|   2 +-
 .../regionserver/BusyRegionSplitPolicy.java | 157 +++
 .../regionserver/TestRegionSplitPolicy.java |  51 ++
 3 files changed, 209 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3abd52bd/hbase-common/src/main/resources/hbase-default.xml
--
diff --git a/hbase-common/src/main/resources/hbase-default.xml 
b/hbase-common/src/main/resources/hbase-default.xml
index 55ac497..6010b7a 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -322,7 +322,7 @@ possible configurations would overwhelm and obscure the 
important.
 
org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy
 
   A split policy determines when a region should be split. The various 
other split policies that
-  are available currently are ConstantSizeRegionSplitPolicy, 
DisabledRegionSplitPolicy,
+  are available currently are BusyRegionSplitPolicy, 
ConstantSizeRegionSplitPolicy, DisabledRegionSplitPolicy,
   DelimitedKeyPrefixRegionSplitPolicy, and KeyPrefixRegionSplitPolicy.
   DisabledRegionSplitPolicy blocks manual region splitting.
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/3abd52bd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BusyRegionSplitPolicy.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BusyRegionSplitPolicy.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BusyRegionSplitPolicy.java
new file mode 100644
index 000..81c468f
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/BusyRegionSplitPolicy.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * This class represents a split policy which makes the split decision based
+ * on how busy a region is. The metric that is used here is the fraction of
+ * total write requests that are blocked due to high memstore utilization.
+ * This fractional rate is calculated over a running window of
+ * "hbase.busy.policy.aggWindow" milliseconds. The rate is a time-weighted
+ * aggregated average of the rate in the current window and the
+ * true average rate in the previous window.
+ *
+ */
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
+public class BusyRegionSplitPolicy extends 
IncreasingToUpperBoundRegionSplitPolicy {
+
+  private static final Log LOG = 
LogFactory.getLog(BusyRegionSplitPolicy.class);
+
+  // Maximum fraction blocked write requests before region is considered for 
split
+  private float maxBlockedRequests;
+  public static final float DEFAULT_MAX_BLOCKED_REQUESTS = 0.2f;
+
+  // Minimum age of the region in milliseconds before it is considered for 
split
+  private long minAge = -1;

[1/2] hbase git commit: HBASE-15974 Create a ReplicationQueuesClientHBaseImpl

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master ae5fe1e61 -> 2093aadec


http://git-wip-us.apache.org/repos/asf/hbase/blob/2093aade/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
index bd6d070..346ff37 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateHBaseImpl.java
@@ -39,6 +39,7 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -58,15 +59,20 @@ public class TestReplicationStateHBaseImpl {
   private static ReplicationQueues rq1;
   private static ReplicationQueues rq2;
   private static ReplicationQueues rq3;
+  private static ReplicationQueuesClient rqc;
   private static ReplicationPeers rp;
 
-  private static final String server1 = 
ServerName.valueOf("hostname1.example.org", 1234, 123L)
-  .toString();
+
+  private static final String server0 = 
ServerName.valueOf("hostname0.example.org", 1234, -1L)
+.toString();
+  private static final String server1 = 
ServerName.valueOf("hostname1.example.org", 1234, 1L)
+.toString();
   private static final String server2 = 
ServerName.valueOf("hostname2.example.org", 1234, 1L)
-  .toString();
+.toString();
   private static final String server3 = 
ServerName.valueOf("hostname3.example.org", 1234, 1L)
-  .toString();
+.toString();
 
+  private static DummyServer ds0;
   private static DummyServer ds1;
   private static DummyServer ds2;
   private static DummyServer ds3;
@@ -77,9 +83,9 @@ public class TestReplicationStateHBaseImpl {
 utility.startMiniCluster();
 conf = utility.getConfiguration();
 conf.setClass("hbase.region.replica.replication.ReplicationQueuesType",
-ReplicationQueuesHBaseImpl.class, ReplicationQueues.class);
-conf.setClass("hbase.region.replica.replication.ReplicationQueuesType",
-  ReplicationQueuesHBaseImpl.class, ReplicationQueues.class);
+  TableBasedReplicationQueuesImpl.class, ReplicationQueues.class);
+
conf.setClass("hbase.region.replica.replication.ReplicationQueuesClientType",
+  TableBasedReplicationQueuesClientImpl.class, 
ReplicationQueuesClient.class);
 zkw = HBaseTestingUtility.getZooKeeperWatcher(utility);
 String replicationZNodeName = conf.get("zookeeper.znode.replication", 
"replication");
 replicationZNode = ZKUtil.joinZNode(zkw.baseZNode, replicationZNodeName);
@@ -88,6 +94,9 @@ public class TestReplicationStateHBaseImpl {
   @Before
   public void setUp() {
 try {
+  ds0 = new DummyServer(server0);
+  rqc = ReplicationFactory.getReplicationQueuesClient(new 
ReplicationQueuesClientArguments(
+conf, ds0));
   ds1 = new DummyServer(server1);
   rq1 = ReplicationFactory.getReplicationQueues(new 
ReplicationQueuesArguments(conf, ds1, zkw));
   rq1.init(server1);
@@ -99,9 +108,6 @@ public class TestReplicationStateHBaseImpl {
   rq3.init(server3);
   rp = ReplicationFactory.getReplicationPeers(zkw, conf, zkw);
   rp.init();
-  rp.addPeer("Queue1", new 
ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus1"));
-  rp.addPeer("Queue2", new 
ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus2"));
-  rp.addPeer("Queue3", new 
ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus3"));
 } catch (Exception e) {
   fail("testReplicationStateHBaseConstruction received an exception" + 
e.getMessage());
 }
@@ -165,13 +171,13 @@ public class TestReplicationStateHBaseImpl {
   try {
 rq1.getLogPosition("Queue1", "NotHereWAL");
 fail("Replication queue should have thrown a ReplicationException for 
reading from a " +
-"non-existent WAL");
+  "non-existent WAL");
   } catch (ReplicationException e) {
   }
   try {
 rq1.getLogPosition("NotHereQueue", "NotHereWAL");
 fail("Replication queue should have thrown a ReplicationException for 
reading from a " +
-"non-existent queue");
+  "non-existent queue");
   } catch (ReplicationException e) {
   }
   // Test removing logs
@@ -198,6 +204,13 @@ public class TestReplicationStateHBaseImpl {
   @Test
   public void TestMultipleReplicationQueuesHBaseImpl () {
 try {
+  rp.addPeer("Queue1", new 
ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus1"));
+  rp.addPeer("Queue2", new 
ReplicationPeerConfig().setClusterKey("localhost:2818:/bogus2"));
+  rp.addPeer("Queue3", new 

hbase git commit: HBASE-15958 Implement ClaimQueues on top of HBase

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 108d39a72 -> babdedc1b


HBASE-15958 Implement ClaimQueues on top of HBase

Building on HBase-15883.
Now implementing the claim queues procedure within an HBase table.
Also added UnitTests to test claimQueue.
Peer tracking will still be performed by ZooKeeper though.
Also modified the queueId tracking procedure so we no longer have to perform 
scans over the Replication Table.
This does make our queue naming schema slightly different from 
ReplicationQueuesZKImpl though.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/babdedc1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/babdedc1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/babdedc1

Branch: refs/heads/master
Commit: babdedc1b0f0159eb526fb5c9ee08525de7ce404
Parents: 108d39a
Author: Joseph Hwang 
Authored: Thu May 19 17:14:33 2016 -0700
Committer: Elliott Clark 
Committed: Thu Jun 9 15:05:54 2016 -0700

--
 .../hbase/replication/ReplicationQueues.java|   8 +-
 .../replication/ReplicationQueuesArguments.java |   4 +-
 .../replication/ReplicationQueuesHBaseImpl.java | 485 ---
 .../replication/ReplicationQueuesZKImpl.java|  26 +-
 .../regionserver/ReplicationSourceManager.java  |   7 +-
 .../replication/TestReplicationStateBasic.java  |   6 +-
 .../TestReplicationStateHBaseImpl.java  | 302 +---
 .../replication/TestReplicationStateZKImpl.java |   1 -
 .../TestReplicationSourceManager.java   |  12 +-
 9 files changed, 579 insertions(+), 272 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 809b122..0de0cc8 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -19,8 +19,8 @@
 package org.apache.hadoop.hbase.replication;
 
 import java.util.List;
-import java.util.SortedMap;
-import java.util.SortedSet;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 
@@ -96,10 +96,10 @@ public interface ReplicationQueues {
   /**
* Take ownership for the set of queues belonging to a dead region server.
* @param regionserver the id of the dead region server
-   * @return A SortedMap of the queues that have been claimed, including a 
SortedSet of WALs in
+   * @return A Map of the queues that have been claimed, including a Set of 
WALs in
* each queue. Returns an empty map if no queues were failed-over.
*/
-  SortedMap claimQueues(String regionserver);
+  Map claimQueues(String regionserver);
 
   /**
* Get a list of all region servers that have outstanding replication 
queues. These servers could

http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
index 4907b73..4fdc4e7 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
@@ -56,11 +56,11 @@ public class ReplicationQueuesArguments {
 this.conf = conf;
   }
 
-  public Abortable getAbort() {
+  public Abortable getAbortable() {
 return abort;
   }
 
-  public void setAbort(Abortable abort) {
+  public void setAbortable(Abortable abort) {
 this.abort = abort;
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/babdedc1/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
index 29f0632..34a5289 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesHBaseImpl.java
+++ 

hbase git commit: HBASE-15926 Added ASF headers which were missed in last commit

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 3fe62fe16 -> 6c0216034


HBASE-15926 Added ASF headers which were missed in last commit

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6c021603
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6c021603
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6c021603

Branch: refs/heads/HBASE-14850
Commit: 6c02160342860e264a6a2bb09daaf3338ae3cb8f
Parents: 3fe62fe
Author: Sudeep Sunthankar 
Authored: Thu Jun 2 20:22:37 2016 +1000
Committer: Elliott Clark 
Committed: Tue Jun 7 11:51:32 2016 -0700

--
 hbase-native-client/Makefile| 17 +
 hbase-native-client/Makefile.protos | 17 +
 2 files changed, 34 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6c021603/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 7e68b6a..64cef06 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 #use "gcc" to compile source files
 CC:=g++
 LD:=g++

http://git-wip-us.apache.org/repos/asf/hbase/blob/6c021603/hbase-native-client/Makefile.protos
--
diff --git a/hbase-native-client/Makefile.protos 
b/hbase-native-client/Makefile.protos
index 4f1727a..2c6316b 100644
--- a/hbase-native-client/Makefile.protos
+++ b/hbase-native-client/Makefile.protos
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 #use "gcc" to compile source files
 CC:=g++
 LD:=g++

hbase git commit: HBASE-15883 Adding WAL files and tracking offsets in HBase.

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/master 9a53d8b38 -> 21e98271c


HBASE-15883 Adding WAL files and tracking offsets in HBase.

Implemented ReplicationQueuesHBaseImpl that tracks WAL offsets and replication 
queues in an HBase table.
Only wrote the basic tracking methods, have not implemented claimQueue() or 
HFileRef methods yet.
Wrote a basic unit test for ReplicationQueueHBaseImpl that tests the 
implemented functions on a single Region Server

Signed-off-by: Elliott Clark 
Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/21e98271
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/21e98271
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/21e98271

Branch: refs/heads/master
Commit: 21e98271c32f0d44106515a72b2c92d518c03668
Parents: 9a53d8b
Author: Joseph Hwang 
Authored: Thu May 19 17:14:33 2016 -0700
Committer: Elliott Clark 
Committed: Fri Jun 3 15:23:10 2016 -0700

--
 .../hbase/replication/ReplicationFactory.java   |  11 +-
 .../hbase/replication/ReplicationQueues.java|   8 +-
 .../replication/ReplicationQueuesArguments.java |  66 +++
 .../replication/ReplicationQueuesHBaseImpl.java | 491 +++
 .../replication/ReplicationQueuesZKImpl.java|  13 +-
 .../replication/regionserver/Replication.java   |  12 +-
 .../regionserver/ReplicationSourceManager.java  |   5 +-
 .../replication/TestReplicationAdmin.java   |   3 +-
 .../hbase/master/cleaner/TestLogsCleaner.java   |   3 +-
 .../cleaner/TestReplicationHFileCleaner.java|   4 +-
 .../replication/TestReplicationStateBasic.java  |   2 +-
 .../TestReplicationStateHBaseImpl.java  | 243 +
 .../replication/TestReplicationStateZKImpl.java |  13 +-
 .../TestReplicationSourceManager.java   |  36 +-
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   |   4 +-
 15 files changed, 871 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
index 91e77ca..e264a4d 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.replication;
 
+import org.apache.commons.lang.reflect.ConstructorUtils;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Abortable;
@@ -30,9 +31,11 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 @InterfaceAudience.Private
 public class ReplicationFactory {
 
-  public static ReplicationQueues getReplicationQueues(final ZooKeeperWatcher 
zk,
-  Configuration conf, Abortable abortable) {
-return new ReplicationQueuesZKImpl(zk, conf, abortable);
+  public static ReplicationQueues 
getReplicationQueues(ReplicationQueuesArguments args)
+  throws Exception {
+Class classToBuild = args.getConf().getClass("hbase.region.replica." +
+"replication.ReplicationQueuesType", ReplicationQueuesZKImpl.class);
+return (ReplicationQueues) 
ConstructorUtils.invokeConstructor(classToBuild, args);
   }
 
   public static ReplicationQueuesClient getReplicationQueuesClient(final 
ZooKeeperWatcher zk,
@@ -44,7 +47,7 @@ public class ReplicationFactory {
   Abortable abortable) {
 return getReplicationPeers(zk, conf, null, abortable);
   }
-  
+
   public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher 
zk, Configuration conf,
   final ReplicationQueuesClient queuesClient, Abortable abortable) {
 return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);

http://git-wip-us.apache.org/repos/asf/hbase/blob/21e98271/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0d47a88..db6da91 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -83,13 +83,13 @@ public interface ReplicationQueues {
   /**
* Get a list of all 

hbase git commit: HBASE-15822 Move to the latest docker base image

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 5b10031a1 -> 3fe62fe16


HBASE-15822 Move to the latest docker base image


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3fe62fe1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3fe62fe1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3fe62fe1

Branch: refs/heads/HBASE-14850
Commit: 3fe62fe16d3292b73449775a6d6e4f89b9333b47
Parents: 5b10031
Author: Elliott Clark 
Authored: Tue May 31 10:42:45 2016 -0700
Committer: Elliott Clark 
Committed: Tue May 31 10:42:45 2016 -0700

--
 hbase-native-client/Dockerfile   | 2 +-
 hbase-native-client/if/RPC.proto | 1 +
 hbase-native-client/if/WAL.proto | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3fe62fe1/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index be0cbc6..ca87523 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman:20160425
+FROM pjameson/buck-folly-watchman:20160511
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5

http://git-wip-us.apache.org/repos/asf/hbase/blob/3fe62fe1/hbase-native-client/if/RPC.proto
--
diff --git a/hbase-native-client/if/RPC.proto b/hbase-native-client/if/RPC.proto
index 59bb03d..8413d25 100644
--- a/hbase-native-client/if/RPC.proto
+++ b/hbase-native-client/if/RPC.proto
@@ -125,6 +125,7 @@ message RequestHeader {
   // 0 is NORMAL priority.  200 is HIGH.  If no priority, treat it as NORMAL.
   // See HConstants.
   optional uint32 priority = 6;
+  optional uint32 timeout = 7;
 }
 
 message ResponseHeader {

http://git-wip-us.apache.org/repos/asf/hbase/blob/3fe62fe1/hbase-native-client/if/WAL.proto
--
diff --git a/hbase-native-client/if/WAL.proto b/hbase-native-client/if/WAL.proto
index cb9bd8f..2061b22 100644
--- a/hbase-native-client/if/WAL.proto
+++ b/hbase-native-client/if/WAL.proto
@@ -44,9 +44,9 @@ message WALKey {
   required uint64 log_sequence_number = 3;
   required uint64 write_time = 4;
   /*
-  This parameter is deprecated in favor of clusters which 
+  This parameter is deprecated in favor of clusters which
   contains the list of clusters that have consumed the change.
-  It is retained so that the log created by earlier releases (0.94) 
+  It is retained so that the log created by earlier releases (0.94)
   can be read by the newer releases.
   */
   optional UUID cluster_id = 5 [deprecated=true];
@@ -132,6 +132,7 @@ message StoreDescriptor {
   required bytes family_name = 1;
   required string store_home_dir = 2; //relative to region dir
   repeated string store_file = 3; // relative to store dir
+  optional uint64 store_file_size_bytes = 4; // size of store file
 }
 
 /**

hbase git commit: HBASE-15851 Makefile update for build env

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 f49f262f3 -> 5b10031a1


HBASE-15851 Makefile update for build env

1) Makefile to compile protobuf sources which are extracted in build
2) Added -O2 and -D_GLIBCXX_USE_CXX11_ABI=0 compilation flags
3) Header files added in Makefile


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5b10031a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5b10031a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5b10031a

Branch: refs/heads/HBASE-14850
Commit: 5b10031a1b1821a4403f63db1728e9f380af14f5
Parents: f49f262
Author: sudeeps 
Authored: Sun May 22 09:38:47 2016 +1000
Committer: Elliott Clark 
Committed: Fri May 27 11:32:43 2016 -0700

--
 hbase-native-client/Makefile  | 132 +++--
 hbase-native-client/Makefile.protos   |  53 
 hbase-native-client/core/meta-utils.h |   2 +-
 3 files changed, 159 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5b10031a/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 826233f..7e68b6a 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,37 +1,115 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-build:
-   $(shell buck build core/... )
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+ 
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+PROTO_SRC_DIR = build/if
+MODULES = connection core serde test-util utils
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = . build/
 
-check:
-   $(shell buck test --all --no-results-cache )
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 
-pedantic -fPIC
+LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
+LINKFLAG = -shared
 
-doc:
-   $(shell doxygen hbase.doxygen > /dev/null )
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard 
if/*.proto))
+DEPS =  $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard 
if/*.proto))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+   
+LIB_DIR = /usr/local
+LIB_LIBDIR = $(LIB_DIR)/lib
+LIB_INCDIR = $(LIB_DIR)/include
+LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so
+ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a
+LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so
+ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
+
+vpath %.cc $(SRC_DIR)
+
+$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+   $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(LIB_RELEASE): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+   $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES) 
+endef
+
+.PHONY: all clean install 
+
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) 
$(ARC_RELEASE) 
+
+checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+
+protos: createprotosrc
+   @make all -f Makefile.protos
+
+createprotosrc:$(PROTO_SRC_DIR)
+   @protoc --proto_path=if --cpp_out=$(PROTO_SRC_DIR) if/*.proto
+
+install:
+   cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
+   cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
+   cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
+   cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+   

hbase git commit: HBASE-15471 Added in Priority, General, and Replication queue sizes for RegionServers to the Web UI

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 627b48b79 -> 2346b5e21


HBASE-15471 Added in Priority, General, and Replication queue sizes for 
RegionServers to the Web UI

Changed UI labels so that queue "size" refers to size in bytes and queue 
"length" refers to number of items in queue.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2346b5e2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2346b5e2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2346b5e2

Branch: refs/heads/branch-1
Commit: 2346b5e2169c802ed77554f4a8a79cd97c07ed3f
Parents: 627b48b
Author: Joseph Hwang 
Authored: Tue May 17 15:52:26 2016 -0700
Committer: Elliott Clark 
Committed: Wed May 25 15:15:01 2016 -0700

--
 .../hbase/tmpl/regionserver/ServerMetricsTmpl.jamon | 12 +---
 1 file changed, 9 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2346b5e2/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
index 29f0791..2305f2e 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
@@ -191,14 +191,20 @@ MetricsHBaseServerWrapper mServerWrap;
 
 
 
-Compaction Queue Size
-Flush Queue Size
-Call Queue Size (bytes)
+Compaction Queue Length
+Flush Queue Length
+Priority Call Queue Length
+General Call Queue Length
+Replication Call Queue Length
+Total Call Queue Size (bytes)
 
 
 
 <% mWrap.getCompactionQueueSize() %>
 <% mWrap.getFlushQueueSize() %>
+<% mServerWrap.getPriorityQueueLength() %>
+<% mServerWrap.getGeneralQueueLength() %>
+<% mServerWrap.getReplicationQueueLength() %>
 <% 
TraditionalBinaryPrefix.long2String(mServerWrap.getTotalQueueSize(), "B", 1) 
%>
 
 

hbase git commit: HBASE-15471 Added in Priority, General, and Replication queue sizes for RegionServers to the Web UI

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 37e080d70 -> b2e8a4158


HBASE-15471 Added in Priority, General, and Replication queue sizes for 
RegionServers to the Web UI

Changed UI labels so that queue "size" refers to size in bytes and queue 
"length" refers to number of items in queue.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b2e8a415
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b2e8a415
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b2e8a415

Branch: refs/heads/branch-1.3
Commit: b2e8a415887e22decd4191876b34f1875b987776
Parents: 37e080d
Author: Joseph Hwang 
Authored: Tue May 17 15:52:26 2016 -0700
Committer: Elliott Clark 
Committed: Wed May 25 15:15:11 2016 -0700

--
 .../hbase/tmpl/regionserver/ServerMetricsTmpl.jamon | 12 +---
 1 file changed, 9 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b2e8a415/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
--
diff --git 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
index 29f0791..2305f2e 100644
--- 
a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
+++ 
b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
@@ -191,14 +191,20 @@ MetricsHBaseServerWrapper mServerWrap;
 
 
 
-Compaction Queue Size
-Flush Queue Size
-Call Queue Size (bytes)
+Compaction Queue Length
+Flush Queue Length
+Priority Call Queue Length
+General Call Queue Length
+Replication Call Queue Length
+Total Call Queue Size (bytes)
 
 
 
 <% mWrap.getCompactionQueueSize() %>
 <% mWrap.getFlushQueueSize() %>
+<% mServerWrap.getPriorityQueueLength() %>
+<% mServerWrap.getGeneralQueueLength() %>
+<% mServerWrap.getReplicationQueueLength() %>
 <% 
TraditionalBinaryPrefix.long2String(mServerWrap.getTotalQueueSize(), "B", 1) 
%>
 
 

[13/42] hbase git commit: HBASE-14853 Add on protobuf to c++ chain

[eclark]

http://git-wip-us.apache.org/repos/asf/hbase/blob/2c911aeb/hbase-native-client/if/Master.proto
--
diff --git a/hbase-native-client/if/Master.proto 
b/hbase-native-client/if/Master.proto
new file mode 100644
index 000..4d3a2e1
--- /dev/null
+++ b/hbase-native-client/if/Master.proto
@@ -0,0 +1,778 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// All to do with the Master.  Includes schema management since these
+// changes are run by the Master process.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+import "ClusterStatus.proto";
+import "ErrorHandling.proto";
+import "Procedure.proto";
+import "Quota.proto";
+
+/* Column-level protobufs */
+
+message AddColumnRequest {
+  required TableName table_name = 1;
+  required ColumnFamilySchema column_families = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message AddColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DeleteColumnRequest {
+  required TableName table_name = 1;
+  required bytes column_name = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message DeleteColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyColumnRequest {
+  required TableName table_name = 1;
+  required ColumnFamilySchema column_families = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+/* Region-level Protos */
+
+message MoveRegionRequest {
+  required RegionSpecifier region = 1;
+  optional ServerName dest_server_name = 2;
+}
+
+message MoveRegionResponse {
+}
+
+/**
+ * Dispatch merging the specified regions.
+ */
+message DispatchMergingRegionsRequest {
+  required RegionSpecifier region_a = 1;
+  required RegionSpecifier region_b = 2;
+  optional bool forcible = 3 [default = false];
+}
+
+message DispatchMergingRegionsResponse {
+}
+
+message AssignRegionRequest {
+  required RegionSpecifier region = 1;
+}
+
+message AssignRegionResponse {
+}
+
+message UnassignRegionRequest {
+  required RegionSpecifier region = 1;
+  optional bool force = 2 [default = false];
+}
+
+message UnassignRegionResponse {
+}
+
+message OfflineRegionRequest {
+  required RegionSpecifier region = 1;
+}
+
+message OfflineRegionResponse {
+}
+
+/* Table-level protobufs */
+
+message CreateTableRequest {
+  required TableSchema table_schema = 1;
+  repeated bytes split_keys = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message CreateTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DeleteTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message TruncateTableRequest {
+  required TableName tableName = 1;
+  optional bool preserveSplits = 2 [default = false];
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message TruncateTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message EnableTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message EnableTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DisableTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message DisableTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyTableRequest {
+  required TableName table_name = 1;
+  required TableSchema table_schema = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyTableResponse 

[07/42] hbase git commit: HBASE-15850 Localize the configuration change in testCheckTableLocks to reduce flakiness of TestHBaseFsck test suite (Stephen Yuan Jiang)

[eclark]

HBASE-15850 Localize the configuration change in testCheckTableLocks to reduce 
flakiness of TestHBaseFsck test suite (Stephen Yuan Jiang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8aa8abfc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8aa8abfc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8aa8abfc

Branch: refs/heads/HBASE-14850
Commit: 8aa8abfcb672e4b905c487a10f1511db431f7006
Parents: 5ac54e6
Author: Stephen Yuan Jiang 
Authored: Wed May 18 09:41:13 2016 -0700
Committer: Stephen Yuan Jiang 
Committed: Wed May 18 13:22:41 2016 -0700

--
 .../hadoop/hbase/util/TestHBaseFsckOneRS.java   | 16 ++--
 1 file changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8aa8abfc/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
index 165fea6..57bc77e 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
@@ -1472,7 +1472,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck 
{
 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table 
lock expire
 
 hbck = doFsck(conf, false);
-assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] 
{HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK});
+assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
+HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK});
 
 final CountDownLatch latch = new CountDownLatch(1);
 new Thread() {
@@ -1496,24 +1497,27 @@ public class TestHBaseFsckOneRS extends 
BaseTestHBaseFsck {
 Threads.sleep(300); // wait some more to ensure writeLock.acquire() is 
called
 
 hbck = doFsck(conf, false);
+// still one expired, one not-expired
 assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
-HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK}); // still one 
expired, one not-expired
+HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK});
 
 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table 
lock expire
 
 hbck = doFsck(conf, false);
-assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] 
{HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK,
+assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
+HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK,
 HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK}); // both are 
expired
 
-conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
+Configuration localConf = new Configuration(conf);
 // reaping from ZKInterProcessWriteLock uses znode cTime,
 // which is not injectable through EnvironmentEdge
+localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
 
 Threads.sleep(10);
-hbck = doFsck(conf, true); // now fix both cases
+hbck = doFsck(localConf, true); // now fix both cases
 
-hbck = doFsck(conf, false);
+hbck = doFsck(localConf, false);
 assertNoErrors(hbck);
 
 // ensure that locks are deleted

[18/42] hbase git commit: HBASE-14854 Read meta location from zk

[eclark]

HBASE-14854 Read meta location from zk


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b9881d21
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b9881d21
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b9881d21

Branch: refs/heads/HBASE-14850
Commit: b9881d21beedc1354094d558eb6458ecfdd51cc1
Parents: dd8bb5c
Author: Elliott Clark 
Authored: Sat Mar 5 00:09:08 2016 -0800
Committer: Elliott Clark 
Committed: Wed May 18 15:48:52 2016 -0700

--
 hbase-native-client/Dockerfile  |  20 ++-
 hbase-native-client/bin/start-docker.sh |   5 +-
 hbase-native-client/core/BUCK   | 106 +++---
 .../core/HBaseNativeClientTestEnv.cc|  42 --
 .../core/SampleNativeClientTest.cc  |  28 
 hbase-native-client/core/location-cache-test.cc |  14 ++
 hbase-native-client/core/location-cache.cc  |  67 +
 hbase-native-client/core/location-cache.h   |  35 +
 .../core/native-client-test-env.cc  |  42 ++
 .../core/simple-native-client-test.cc   |  28 
 hbase-native-client/core/test_env.h |   2 +
 hbase-native-client/if/BUCK |   1 +
 hbase-native-client/third-party/BUCK| 138 ++-
 13 files changed, 339 insertions(+), 189 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1364d22..36959a5 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM cpp_update 
+FROM pjameson/buck-folly-watchman
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5
@@ -25,20 +25,26 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g 
-fno-omit-frame-pointer -O3 -p
 RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
 RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git 
/usr/src/protobuf && \
   cd /usr/src/protobuf/ && \
+  ldconfig && \
   ./autogen.sh && \
-  ./configure --disable-shared && \
+  ./configure && \
   make && \
-  make check && \
-  make install
+  make install && \ 
+  make clean && \
+  rm -rf .git
+
 RUN cd /usr/src && \
   wget 
http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz 
&& \ 
   tar zxf zookeeper-3.4.8.tar.gz && \ 
   rm -rf zookeeper-3.4.8.tar.gz && \
   cd zookeeper-3.4.8 && \
   cd src/c && \
-  ./configure --disable-shared && \
+  ldconfig && \
+  ./configure && \
   make && \
   make install && \
-  make clean 
+  make clean
+
+RUN ldconfig
 
-WORKDIR /usr/local/src/hbase/hbase-native-client
+WORKDIR /usr/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 4426705..725ed6a 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -19,8 +19,11 @@
 set -e
 set -x
 
+# Try out some standard docker machine names that could work
 eval "$(docker-machine env docker-vm)"
 eval "$(docker-machine env dinghy)"
+
+# Build the image
 docker build -t hbase_native .
 
 
@@ -36,6 +39,6 @@ fi;
 
 docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-   -v ${PWD}/..:/usr/local/src/hbase \
+   -v ${PWD}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash

http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index ef027a1..817b5a0 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -15,52 +15,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+cxx_library(name="core",
+headers=[
+"admin.h",
+"client.h",
+"connection.h",
+"connection_attr.h",
+"delete.h",
+"get.h",
+"hbase_macros.h",
+"mutation.h",
+"put.h",
+"scanner.h",
+"location-cache.h",
+],
+srcs=[
+"admin.cc",
+"client.cc",
+

[20/42] hbase git commit: HBASE-15604 Add a good readme on the build.

[eclark]

HBASE-15604 Add a good readme on the build.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/711b6619
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/711b6619
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/711b6619

Branch: refs/heads/HBASE-14850
Commit: 711b6619314b6e8581d28a551b4d3b54c1bc8fb4
Parents: 80977a6
Author: Elliott Clark 
Authored: Fri Apr 8 13:44:45 2016 -0700
Committer: Elliott Clark 
Committed: Wed May 18 15:48:52 2016 -0700

--
 hbase-native-client/BUILDING.md | 63 
 1 file changed, 63 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/711b6619/hbase-native-client/BUILDING.md
--
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
new file mode 100644
index 000..20ef2a0
--- /dev/null
+++ b/hbase-native-client/BUILDING.md
@@ -0,0 +1,63 @@
+
+
+#Building HBase native client
+
+The HBase native client build using buck and produces a linux library.
+
+
+# Dependencies
+
+The easiest way to build hbase-native-client is to
+use [Docker](https://www.docker.com/). This will mean that any platform
+with docker can be used to build the hbase-native-client. It will also
+mean that you're building with the same versions of all dependencies that
+the client is tested with.
+
+On OSX the current boot2docker solution will work however it's pretty
+slow. If things get too slow using 
[dinghy](https://github.com/codekitchen/dinghy)
+can help speed things up by using nfs.
+
+If possible pairing virtual box with dinghy will result in the fastest,
+most stable docker environment. However none of it is needed.
+
+# Building using docker
+
+To start with make sure that you have built the java project using
+`mvn package -DskipTests`. That will allow all tests to spin up a standalone
+hbase instance from the jar's created.
+
+Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
+that will build the docker development environment and when complete will
+drop you into a shell on a linux vm with all the tools needed installed.
+
+# Buck
+
+From then on we can use [buck](https://buckbuild.com/) to build everything.
+For example:
+```
+buck build //core:core
+buck test --all
+buck build //core:simple-client
+```
+
+That will build the library, then build and test everything, then build
+the simple-client binary. Buck will find all modules used, and compile 
+them in parallel, caching the results. Output from buck is in the buck-out
+foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs

[37/42] hbase git commit: HBASE-15777 Fix needs header in client handler

[eclark]

HBASE-15777 Fix needs header in client handler


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7d7bc06e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7d7bc06e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7d7bc06e

Branch: refs/heads/HBASE-14850
Commit: 7d7bc06e6a69874133be67107cc712c93faf66ce
Parents: 285b114
Author: Elliott Clark <ecl...@apache.org>
Authored: Thu May 5 13:51:18 2016 -0700
Committer: Elliott Clark <elli...@fb.com>
Committed: Wed May 18 15:48:52 2016 -0700

--
 .../connection/client-handler.cc| 31 
 hbase-native-client/connection/client-handler.h | 19 +++-
 2 files changed, 36 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7d7bc06e/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index b92ad89..4fdb7ae 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
 using google::protobuf::Message;
 
 ClientHandler::ClientHandler(std::string user_name)
-: user_name_(user_name), need_send_header_(true), serde_(),
+: user_name_(user_name), serde_(), header_info_(),
   resp_msgs_(
   make_unique<folly::AtomicHashMap<
   uint32_t, std::shared_ptr>>(5000)) {}
@@ -81,22 +81,27 @@ void ClientHandler::read(Context *ctx, 
std::unique_ptr buf) {
   }
 }
 
-// TODO(eclark): Figure out how to handle the
-// network errors that are going to come.
 Future ClientHandler::write(Context *ctx, std::unique_ptr r) {
   // Keep track of if we have sent the header.
-  if (UNLIKELY(need_send_header_)) {
-need_send_header_ = false;
+  //
+  // even though the bool is atomic we can load it lazily here.
+  if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
 
-// Should we be sending just one fireWrite?
-// Right now we're sending one for the header
-// and one for the request.
+// Grab the lock.
+// We need to make sure that no one gets past here without there being a
+// hearder sent.
+std::lock_guard lock(header_info_->mutex_);
+
+// Now see if we are the first thread to get through.
 //
-// That doesn't seem like too bad, but who knows.
-auto pre = serde_.Preamble();
-auto header = serde_.Header(user_name_);
-pre->appendChain(std::move(header));
-ctx->fireWrite(std::move(pre));
+// If this is the first thread to get through then the
+// need_send_header will have been true before this.
+if (header_info_->need_.exchange(false)) {
+  auto pre = serde_.Preamble();
+  auto header = serde_.Header(user_name_);
+  pre->appendChain(std::move(header));
+  ctx->fireWrite(std::move(pre));
+}
   }
 
   resp_msgs_->insert(r->call_id(), r->resp_msg());

http://git-wip-us.apache.org/repos/asf/hbase/blob/7d7bc06e/hbase-native-client/connection/client-handler.h
--
diff --git a/hbase-native-client/connection/client-handler.h 
b/hbase-native-client/connection/client-handler.h
index be5143c..1a4275f 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -21,6 +21,8 @@
 #include 
 #include 
 
+#include 
+#include 
 #include 
 
 #include "serde/rpc.h"
@@ -29,6 +31,7 @@
 namespace hbase {
 class Request;
 class Response;
+class HeaderInfo;
 }
 namespace google {
 namespace protobuf {
@@ -37,6 +40,7 @@ class Message;
 }
 
 namespace hbase {
+
 class ClientHandler : public wangle::Handler<std::unique_ptr,
  Response, 
std::unique_ptr,
  std::unique_ptr> {
@@ -47,7 +51,7 @@ public:
std::unique_ptr r) override;
 
 private:
-  bool need_send_header_;
+  std::unique_ptr header_info_;
   std::string user_name_;
   RpcSerde serde_;
 
@@ -56,4 +60,17 @@ private:
   uint32_t, std::shared_ptr>>
   resp_msgs_;
 };
+
+/**
+ * Class to contain the info about if the connection header and preamble has
+ * been sent.
+ *
+ * We use a serperate class here so that ClientHandler is relocatable.
+ */
+class HeaderInfo {
+public:
+  HeaderInfo() : need_(true), mutex_() {}
+  std::atomic need_;
+  std::mutex mutex_;
+};
 } // namespace hbase