hbase git commit: HBASE-16081 Wait for Replication Tasks to complete before killing the ThreadPoolExecutor inside of HBaseInterClusterReplicationEndpoint

[antonov]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 00c91b01e -> 0fda2bc9e


HBASE-16081 Wait for Replication Tasks to complete before killing the 
ThreadPoolExecutor inside of HBaseInterClusterReplicationEndpoint

Signed-off-by: Mikhail Antonov 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0fda2bc9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0fda2bc9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0fda2bc9

Branch: refs/heads/branch-1.3
Commit: 0fda2bc9e7cbd58d4e67d0e9dcc420bc7ea98eab
Parents: 00c91b0
Author: Joseph Hwang 
Authored: Mon Jul 11 13:17:56 2016 -0700
Committer: Mikhail Antonov 
Committed: Mon Jul 11 18:10:54 2016 -0700

--
 .../hbase/replication/ReplicationEndpoint.java  |  7 +++-
 .../HBaseInterClusterReplicationEndpoint.java   | 34 +---
 .../regionserver/ReplicationSourceManager.java  |  3 +-
 3 files changed, 36 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0fda2bc9/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
index ac1257f..a88e454 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -58,6 +59,7 @@ public interface ReplicationEndpoint extends Service {
 private final String peerId;
 private final UUID clusterId;
 private final MetricsSource metrics;
+private final Abortable abortable;
 
 @InterfaceAudience.Private
 public Context(
@@ -68,7 +70,8 @@ public interface ReplicationEndpoint extends Service {
 final UUID clusterId,
 final ReplicationPeer replicationPeer,
 final MetricsSource metrics,
-final TableDescriptors tableDescriptors) {
+final TableDescriptors tableDescriptors,
+final Abortable abortable) {
   this.peerConfig = peerConfig;
   this.conf = conf;
   this.fs = fs;
@@ -77,6 +80,7 @@ public interface ReplicationEndpoint extends Service {
   this.replicationPeer = replicationPeer;
   this.metrics = metrics;
   this.tableDescriptors = tableDescriptors;
+  this.abortable = abortable;
 }
 public Configuration getConfiguration() {
   return conf;
@@ -102,6 +106,7 @@ public interface ReplicationEndpoint extends Service {
 public TableDescriptors getTableDescriptors() {
   return tableDescriptors;
 }
+public Abortable getAbortable() { return abortable; }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/0fda2bc9/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index b94d21d..548f716 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableNotFoundException;
@@ -71,17 +72,19 @@ import com.google.common.annotations.VisibleForTesting;
 public class HBaseInterClusterReplicationEndpoint extends 
HBaseReplicationEndpoint {
 
   private static final Log LOG = 
LogFactory.getLog(HBaseInterClusterReplicationEndpoint.class);
-  private HConnection conn;
 
-  private Configuration conf;
+  private static final long DEFAULT_MAX_TERMINATION_WAIT_MULTIPLIER = 2;
 
+  private HConnection conn;
+  private Configuration conf;
   // How long should we 

hbase git commit: HBASE-16081 Wait for Replication Tasks to complete before killing the ThreadPoolExecutor inside of HBaseInterClusterReplicationEndpoint

[antonov]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 3ff6f4585 -> 7fa311a94


HBASE-16081 Wait for Replication Tasks to complete before killing the 
ThreadPoolExecutor inside of HBaseInterClusterReplicationEndpoint

Signed-off-by: Mikhail Antonov 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7fa311a9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7fa311a9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7fa311a9

Branch: refs/heads/branch-1
Commit: 7fa311a9408ab8d1028d1a788aa88f65da447628
Parents: 3ff6f45
Author: Joseph Hwang 
Authored: Mon Jul 11 13:17:56 2016 -0700
Committer: Mikhail Antonov 
Committed: Mon Jul 11 17:54:24 2016 -0700

--
 .../hbase/replication/ReplicationEndpoint.java  |  7 +++-
 .../HBaseInterClusterReplicationEndpoint.java   | 34 +---
 .../regionserver/ReplicationSourceManager.java  |  2 +-
 3 files changed, 36 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7fa311a9/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
index c92b53d..69db31c 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationEndpoint.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
 
+import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -57,6 +58,7 @@ public interface ReplicationEndpoint extends Service, 
ReplicationPeerConfigListe
 private final String peerId;
 private final UUID clusterId;
 private final MetricsSource metrics;
+private final Abortable abortable;
 
 @InterfaceAudience.Private
 public Context(
@@ -66,7 +68,8 @@ public interface ReplicationEndpoint extends Service, 
ReplicationPeerConfigListe
 final UUID clusterId,
 final ReplicationPeer replicationPeer,
 final MetricsSource metrics,
-final TableDescriptors tableDescriptors) {
+final TableDescriptors tableDescriptors,
+final Abortable abortable) {
   this.conf = conf;
   this.fs = fs;
   this.clusterId = clusterId;
@@ -74,6 +77,7 @@ public interface ReplicationEndpoint extends Service, 
ReplicationPeerConfigListe
   this.replicationPeer = replicationPeer;
   this.metrics = metrics;
   this.tableDescriptors = tableDescriptors;
+  this.abortable = abortable;
 }
 public Configuration getConfiguration() {
   return conf;
@@ -99,6 +103,7 @@ public interface ReplicationEndpoint extends Service, 
ReplicationPeerConfigListe
 public TableDescriptors getTableDescriptors() {
   return tableDescriptors;
 }
+public Abortable getAbortable() { return abortable; }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/7fa311a9/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
index b94d21d..548f716 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HBaseInterClusterReplicationEndpoint.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableNotFoundException;
@@ -71,17 +72,19 @@ import com.google.common.annotations.VisibleForTesting;
 public class HBaseInterClusterReplicationEndpoint extends 
HBaseReplicationEndpoint {
 
   private static final Log LOG = 
LogFactory.getLog(HBaseInterClusterReplicationEndpoint.class);
-  private HConnection conn;
 
-  private Configuration conf;
+  private static final long 

[37/50] [abbrv] hbase git commit: HBASE-15748 Don't link in static libunwind.

[eclark]

HBASE-15748 Don't link in static libunwind.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50d16726
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50d16726
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50d16726

Branch: refs/heads/HBASE-14850
Commit: 50d16726ccf5b93a318417ab5b69d083c6f6ca8d
Parents: f74379e
Author: Elliott Clark 
Authored: Mon May 2 11:41:02 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/third-party/BUCK | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/50d16726/hbase-native-client/third-party/BUCK
--
diff --git a/hbase-native-client/third-party/BUCK 
b/hbase-native-client/third-party/BUCK
index 4327530..6561b2a 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -51,7 +51,7 @@ def add_dynamic_libs(names=[]):
 return rules
 
 
-system_libs = ["unwind", "lzma", "event", ]
+system_libs = ["lzma", "event", ]
 local_libs = [
 "double-conversion",
 "boost_regex",
@@ -65,7 +65,7 @@ local_libs = [
 "glog",
 "protobuf",
 ]
-dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ]
+dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", "unwind"]
 dynamic_rules = add_dynamic_libs(dynamic_libs)
 tp_dep_rules =  add_system_libs(system_libs,) \
   + add_system_libs(local_libs, lib_dir = "/usr/local/lib") \

[44/50] [abbrv] hbase git commit: HBASE-15730 Add on script to format all .h, .cc, and BUCK files.

[eclark]

HBASE-15730 Add on script to format all .h,.cc, and BUCK files.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1140d74
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1140d74
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1140d74

Branch: refs/heads/HBASE-14850
Commit: c1140d742b5fa0ec6270479f580d58b0a45c2a17
Parents: 14a33d7
Author: Elliott Clark 
Authored: Wed Apr 27 15:52:27 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/format-code.sh  |  7 +++
 hbase-native-client/connection/request.cc   | 14 +++---
 hbase-native-client/utils/BUCK  |  8 
 hbase-native-client/utils/user-util-test.cc |  2 +-
 hbase-native-client/utils/user-util.cc  |  4 ++--
 hbase-native-client/utils/user-util.h   |  2 +-
 6 files changed, 22 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/bin/format-code.sh
--
diff --git a/hbase-native-client/bin/format-code.sh 
b/hbase-native-client/bin/format-code.sh
new file mode 100755
index 000..24d6515
--- /dev/null
+++ b/hbase-native-client/bin/format-code.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+
+find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 
clang-format -i
+find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/connection/request.cc
--
diff --git a/hbase-native-client/connection/request.cc 
b/hbase-native-client/connection/request.cc
index 50ea029..d4a4603 100644
--- a/hbase-native-client/connection/request.cc
+++ b/hbase-native-client/connection/request.cc
@@ -30,16 +30,16 @@ Request::Request(std::shared_ptr 
req,
 
 std::unique_ptr Request::get() {
   return std::make_unique(std::make_shared(),
-  std::make_shared(),
-  "Get");
+   std::make_shared(),
+   "Get");
 }
 std::unique_ptr Request::mutate() {
-  return 
std::make_unique(std::make_shared(),
-  
std::make_shared(),
-  "Mutate");
+  return std::make_unique(
+  std::make_shared(),
+  std::make_shared(), "Mutate");
 }
 std::unique_ptr Request::scan() {
   return std::make_unique(std::make_shared(),
-  std::make_shared(),
-  "Scan");
+   std::make_shared(),
+   "Scan");
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/BUCK
--
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
index 2b65b12..ad12f7d 100644
--- a/hbase-native-client/utils/BUCK
+++ b/hbase-native-client/utils/BUCK
@@ -23,18 +23,18 @@ cxx_library(name="utils",
 "user-util.cc",
 ],
 deps=[
-  '//third-party:folly',
+'//third-party:folly',
 ],
 visibility=[
 'PUBLIC',
 ],
 tests=[
-  ":user-util-test"
-],)
+":user-util-test"
+], )
 cxx_test(name="user-util-test",
  srcs=[
  "user-util-test.cc",
  ],
  deps=[
  ":utils",
- ],)
+ ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util-test.cc
--
diff --git a/hbase-native-client/utils/user-util-test.cc 
b/hbase-native-client/utils/user-util-test.cc
index 1e72bcc..7c11d8c 100644
--- a/hbase-native-client/utils/user-util-test.cc
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -17,9 +17,9 @@
  *
  */
 
+#include 
 #include 
 #include 
-#include 
 
 #include "utils/user-util.h"
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util.cc
--
diff --git a/hbase-native-client/utils/user-util.cc 
b/hbase-native-client/utils/user-util.cc
index c4427e3..b801537 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -19,10 +19,10 @@
 
 #include "utils/user-util.h"
 
+#include 
 #include 
-#include 
 #include 
-#include 
+#include 
 
 using 

[48/50] [abbrv] hbase git commit: HBASE-15770 Stop using wangle's global executor

[eclark]

HBASE-15770 Stop using wangle's global executor

Summary: Connection pool and connection factory now get thread pools through 
their constructor. This means that the client has the whole control over the 
threads.

Test Plan: simple-client still writes.

Differential Revision: https://reviews.facebook.net/D57801


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3a6efd7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3a6efd7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3a6efd7

Branch: refs/heads/HBASE-14850
Commit: d3a6efd717241188a2541efdb4658fccd3c319f1
Parents: 297a4df
Author: Elliott Clark 
Authored: Fri May 6 14:32:16 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-dispatcher.h  |  3 ++-
 .../connection/client-handler.cc|  2 +-
 .../connection/connection-factory.cc|  8 +++---
 .../connection/connection-factory.h |  2 +-
 .../connection/connection-pool-test.cc  |  1 +
 .../connection/connection-pool.cc   |  7 ++---
 .../connection/connection-pool.h|  2 +-
 hbase-native-client/core/client.cc  | 23 +++-
 hbase-native-client/core/client.h   |  9 +--
 hbase-native-client/core/location-cache-test.cc |  5 ++--
 hbase-native-client/core/location-cache.cc  | 28 +++-
 hbase-native-client/core/location-cache.h   |  8 --
 hbase-native-client/core/simple-client.cc   |  8 +++---
 hbase-native-client/serde/server-name-test.cc   |  1 -
 14 files changed, 63 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
index 4bfb35d..2497cc7 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -31,7 +31,8 @@
 
 namespace hbase {
 /**
- * Dispatcher that assigns a call_id and then routes the response back to the 
future.
+ * Dispatcher that assigns a call_id and then routes the response back to the
+ * future.
  */
 class ClientDispatcher
 : public wangle::ClientDispatcherBase>(5000)) {}

http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-factory.cc
--
diff --git a/hbase-native-client/connection/connection-factory.cc 
b/hbase-native-client/connection/connection-factory.cc
index 635d12d..beec6d5 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -19,8 +19,6 @@
 
 #include "connection/connection-factory.h"
 
-#include 
-
 #include "connection/client-dispatcher.h"
 #include "connection/pipeline.h"
 #include "connection/service.h"
@@ -28,9 +26,9 @@
 using namespace folly;
 using namespace hbase;
 
-ConnectionFactory::ConnectionFactory()
-: io_pool_(std::static_pointer_cast(
-  wangle::getIOExecutor())),
+ConnectionFactory::ConnectionFactory(
+std::shared_ptr io_pool)
+: io_pool_(io_pool),
   pipeline_factory_(std::make_shared()) {}
 
 std::shared_ptr

http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-factory.h
--
diff --git a/hbase-native-client/connection/connection-factory.h 
b/hbase-native-client/connection/connection-factory.h
index 8b6d8d8..fb5d9fe 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -39,7 +39,7 @@ public:
* Constructor.
* There should only 

[27/50] [abbrv] hbase git commit: HBASE-15761 Add on more server name tests

[eclark]

HBASE-15761 Add on more server name tests


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/297a4dff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/297a4dff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/297a4dff

Branch: refs/heads/HBASE-14850
Commit: 297a4dff638649f346e83c8cc685f0009fb3
Parents: fdf0023
Author: Elliott Clark 
Authored: Wed May 4 12:04:18 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/serde/server-name-test.cc | 18 ++
 hbase-native-client/serde/server-name.h   |  4 +++-
 2 files changed, 21 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/297a4dff/hbase-native-client/serde/server-name-test.cc
--
diff --git a/hbase-native-client/serde/server-name-test.cc 
b/hbase-native-client/serde/server-name-test.cc
index 35dcbc1..2281fa2 100644
--- a/hbase-native-client/serde/server-name-test.cc
+++ b/hbase-native-client/serde/server-name-test.cc
@@ -30,3 +30,21 @@ TEST(TestServerName, TestMakeServerName) {
   ASSERT_EQ("test", sn.host_name());
   ASSERT_EQ(123, sn.port());
 }
+
+TEST(TestServerName, TestIps) {
+  auto sn = folly::to("127.0.0.1:999");
+  ASSERT_EQ("127.0.0.1", sn.host_name());
+  ASSERT_EQ(999, sn.port());
+}
+
+TEST(TestServerName, TestThrow) {
+  ASSERT_ANY_THROW(folly::to("Ther's no colon here"));
+}
+
+TEST(TestServerName, TestIPV6) {
+  auto sn = folly::to("[1]:123");
+
+  ASSERT_EQ("[1]", sn.host_name());
+  ASSERT_EQ(123, sn.port());
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/297a4dff/hbase-native-client/serde/server-name.h
--
diff --git a/hbase-native-client/serde/server-name.h 
b/hbase-native-client/serde/server-name.h
index bdba087..9844465 100644
--- a/hbase-native-client/serde/server-name.h
+++ b/hbase-native-client/serde/server-name.h
@@ -12,7 +12,9 @@ template  void parseTo(String in, ServerName 
) {
   std::string s = folly::to(in);
 
   auto delim = s.rfind(":");
-  DCHECK(delim != std::string::npos);
+  if (delim == std::string::npos) {
+throw std::runtime_error("Couldn't parse server name");
+  }
   out.set_host_name(s.substr(0, delim));
   // Now keep everything after the : (delim + 1) to the end.
   out.set_port(folly::to(s.substr(delim + 1)));

[35/50] [abbrv] hbase git commit: HBASE-15851 Makefile update for build env

[eclark]

HBASE-15851 Makefile update for build env

1) Makefile to compile protobuf sources which are extracted in build
2) Added -O2 and -D_GLIBCXX_USE_CXX11_ABI=0 compilation flags
3) Header files added in Makefile


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df14a6a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df14a6a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df14a6a7

Branch: refs/heads/HBASE-14850
Commit: df14a6a7a680dfd15642ab5c420b5da2c8153f88
Parents: 8014cdf
Author: sudeeps 
Authored: Sun May 22 09:38:47 2016 +1000
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Makefile  | 132 +++--
 hbase-native-client/Makefile.protos   |  53 
 hbase-native-client/core/meta-utils.h |   2 +-
 3 files changed, 159 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/df14a6a7/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 826233f..7e68b6a 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,37 +1,115 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-build:
-   $(shell buck build core/... )
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+ 
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+PROTO_SRC_DIR = build/if
+MODULES = connection core serde test-util utils
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = . build/
 
-check:
-   $(shell buck test --all --no-results-cache )
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 
-pedantic -fPIC
+LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
+LINKFLAG = -shared
 
-doc:
-   $(shell doxygen hbase.doxygen > /dev/null )
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard 
if/*.proto))
+DEPS =  $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard 
if/*.proto))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+   
+LIB_DIR = /usr/local
+LIB_LIBDIR = $(LIB_DIR)/lib
+LIB_INCDIR = $(LIB_DIR)/include
+LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so
+ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a
+LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so
+ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
+
+vpath %.cc $(SRC_DIR)
+
+$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+   $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(LIB_RELEASE): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+   $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES) 
+endef
+
+.PHONY: all clean install 
+
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) 
$(ARC_RELEASE) 
+
+checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+
+protos: createprotosrc
+   @make all -f Makefile.protos
+
+createprotosrc:$(PROTO_SRC_DIR)
+   @protoc --proto_path=if --cpp_out=$(PROTO_SRC_DIR) if/*.proto
+
+install:
+   cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
+   cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
+   cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
+   cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+   
+$(PROTO_SRC_DIR):
+   @mkdir -p $@
+
+$(DEBUG_BUILD_DIR):
+   @mkdir -p $@

[45/50] [abbrv] hbase git commit: HBASE-15945 Patch for Cell

[eclark]

HBASE-15945 Patch for Cell

This patch consists of Cell implementation without additional interfaces.

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa3ab420
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa3ab420
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa3ab420

Branch: refs/heads/HBASE-14850
Commit: fa3ab420d19bba853d96576176257b5c12f1a48f
Parents: 2b6341d
Author: Sudeep Sunthankar 
Authored: Mon Jul 4 21:02:25 2016 +1000
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/core/cell-test.cc | 175 +
 hbase-native-client/core/cell.cc  |  77 +
 hbase-native-client/core/cell.h   |  60 ++
 3 files changed, 312 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fa3ab420/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
new file mode 100644
index 000..cbe50eb
--- /dev/null
+++ b/hbase-native-client/core/cell-test.cc
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+
+#include 
+#include 
+#include 
+
+using namespace hbase;
+TEST(CellTest, CellFailureTest) {
+  CellType cell_type = CellType::PUT;
+  std::string row = "row";
+  std::string family = "family";
+  std::string column = "column";
+  std::string value = "value";
+  long timestamp = std::numeric_limits::max();
+  std::string tags = "";
+  std::unique_ptr cell(
+  new Cell(row, family, column, timestamp, value, cell_type));
+  if (cell.get()) {
+EXPECT_NE("row-value", cell.get()->Row());
+EXPECT_NE("family-value", cell.get()->Family());
+EXPECT_NE("column-value", cell.get()->Qualifier());
+EXPECT_NE("value-value", cell.get()->Value());
+EXPECT_NE(8975431260, cell.get()->Timestamp());
+EXPECT_NE(CellType::MAXIMUM, cell.get()->Type());
+  }
+}
+
+TEST(CellTest, CellSuceessTest) {
+  std::string row = "row-value";
+  std::string family = "family-value";
+  std::string column = "column-value";
+  std::string value = "value-value";
+  long timestamp = std::numeric_limits::max();
+  CellType cell_type = CellType::PUT;
+  const std::unique_ptr cell(
+  new Cell(row, family, column, timestamp, value, cell_type));
+  if (cell.get()) {
+EXPECT_EQ(row, cell.get()->Row());
+EXPECT_EQ(family, cell.get()->Family());
+EXPECT_EQ(column, cell.get()->Qualifier());
+EXPECT_EQ(value, cell.get()->Value());
+EXPECT_EQ(timestamp, cell.get()->Timestamp());
+EXPECT_EQ(cell_type, cell.get()->Type());
+  }
+}
+
+TEST(CellTest, MultipleCellsTest) {
+  std::vector cells;
+  for (int i = 0; i < 5; i++) {
+std::string row = "row-value";
+std::string family = "family-value";
+std::string column = "column-value";
+std::string value = "value-value";
+long timestamp = std::numeric_limits::max();
+row += std::to_string(i);
+value += std::to_string(i);
+CellType cell_type = CellType::PUT;
+const Cell *cell = new Cell(row, family, column, timestamp, value,
+cell_type);
+cells.push_back(cell);
+  }
+  int i = 0;
+  for (const auto cell : cells) {
+std::string row = "row-value";
+std::string value = "value-value";
+row += std::to_string(i);
+value += std::to_string(i);
+EXPECT_EQ(row, cell->Row());
+EXPECT_EQ("family-value", cell->Family());
+EXPECT_EQ("column-value", cell->Qualifier());
+EXPECT_EQ(value, cell->Value());
+EXPECT_EQ(std::numeric_limits::max(), cell->Timestamp());
+EXPECT_EQ(CellType::PUT, cell->Type());
+i += 1;
+  }
+  for (const auto cell : cells) {
+delete cell;
+  }
+  cells.clear();
+}
+
+TEST(CellTest, CellRowTest) {
+  std::string row = "only-row";
+  std::string family = "";
+  std::string column = 

[29/50] [abbrv] hbase git commit: HBASE-15620 Add on Call serialization

[eclark]

HBASE-15620 Add on Call serialization

Summary: Add on delimited serialization so that request headers and request 
payloads can be serialized.

Test Plan: Add a unit test.

Differential Revision: https://reviews.facebook.net/D56757


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dc0f0175
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dc0f0175
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dc0f0175

Branch: refs/heads/HBASE-14850
Commit: dc0f0175a3619618318849cb372f74bc7c25a2d0
Parents: 8b2d65f
Author: Elliott Clark 
Authored: Tue Apr 12 12:03:52 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile  |  19 ++-
 hbase-native-client/bin/start-local-hbase.sh|  33 +
 .../bin/start_local_hbase_and_wait.sh   |  33 -
 hbase-native-client/bin/stop-local-hbase.sh |  26 
 .../bin/stop_local_hbase_and_wait.sh|  26 
 hbase-native-client/connection/BUCK |  43 ++
 .../connection/client-dispatcher.cc |  56 
 .../connection/client-dispatcher.h  |  42 ++
 .../connection/client-handler.cc|  80 +++
 hbase-native-client/connection/client-handler.h |  49 +++
 .../connection/connection-factory.cc|  58 
 .../connection/connection-factory.h |  40 ++
 hbase-native-client/connection/pipeline.cc  |  42 ++
 hbase-native-client/connection/pipeline.h   |  39 ++
 hbase-native-client/connection/request.h|  42 ++
 hbase-native-client/connection/response.h   |  46 ++
 hbase-native-client/connection/service.h|  26 
 hbase-native-client/core/BUCK   |  41 +-
 hbase-native-client/core/admin.cc   |  20 ---
 hbase-native-client/core/admin.h|  22 ---
 hbase-native-client/core/client-dispatcher.cc   |  54 ---
 hbase-native-client/core/client-dispatcher.h|  42 --
 .../core/client-serialize-handler.cc| 104 --
 .../core/client-serialize-handler.h |  44 --
 hbase-native-client/core/client.h   |   2 +-
 hbase-native-client/core/connection-factory.cc  |  57 
 hbase-native-client/core/connection-factory.h   |  41 --
 hbase-native-client/core/connection_attr.h  |  24 
 hbase-native-client/core/delete.cc  |  21 ---
 hbase-native-client/core/delete.h   |  27 
 hbase-native-client/core/get-request.h  |   2 +-
 hbase-native-client/core/get-result.h   |   2 +-
 hbase-native-client/core/location-cache.cc  |   3 +-
 hbase-native-client/core/location-cache.h   |   2 +-
 hbase-native-client/core/mutation.cc|  41 --
 hbase-native-client/core/mutation.h |  58 
 .../core/native-client-test-env.cc  |  45 --
 hbase-native-client/core/pipeline.cc|  42 --
 hbase-native-client/core/pipeline.h |  34 -
 hbase-native-client/core/request.h  |  33 -
 hbase-native-client/core/response.h |  34 -
 hbase-native-client/core/scanner.cc |  20 ---
 hbase-native-client/core/scanner.h  |  22 ---
 hbase-native-client/core/service.h  |  26 
 hbase-native-client/core/simple-client.cc   |  45 +-
 .../core/simple-native-client-test.cc   |  25 
 hbase-native-client/core/table-name.h   |   2 +-
 hbase-native-client/core/test-env.cc|  45 ++
 hbase-native-client/if/BUCK |  53 ---
 hbase-native-client/serde/BUCK  |  54 +++
 .../serde/client-deserializer-test.cc   |  67 +
 .../serde/client-deserializer.cc|  68 +
 hbase-native-client/serde/client-deserializer.h |  36 +
 .../serde/client-serializer-test.cc |  75 ++
 hbase-native-client/serde/client-serializer.cc  | 139 +++
 hbase-native-client/serde/client-serializer.h   |  55 
 56 files changed, 1248 insertions(+), 979 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 36959a5..9ccc0d3 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -22,7 +22,14 @@ ARG CXX=/usr/bin/g++-5
 ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
 ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
 
-RUN apt-get 

[47/50] [abbrv] hbase git commit: HBASE-15826 Clean up ASF license issues

[eclark]

HBASE-15826 Clean up ASF license issues


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e718c7dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e718c7dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e718c7dc

Branch: refs/heads/HBASE-14850
Commit: e718c7dc9a4c80015ef7a005714322a2cbb9f6ea
Parents: f6ea493
Author: Elliott Clark 
Authored: Fri May 13 14:53:47 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Makefile | 17 +
 hbase-native-client/bin/copy-protobuf.sh | 16 
 hbase-native-client/bin/format-code.sh   | 16 
 hbase-native-client/bin/start-local-hbase.sh |  1 -
 hbase-native-client/hbase.doxygen| 22 +++---
 hbase-native-client/serde/server-name.h  | 19 +++
 6 files changed, 87 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 90ec6ea..826233f 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 build:
$(shell buck build core/... )
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/copy-protobuf.sh
--
diff --git a/hbase-native-client/bin/copy-protobuf.sh 
b/hbase-native-client/bin/copy-protobuf.sh
index c9e70f2..36541cf 100755
--- a/hbase-native-client/bin/copy-protobuf.sh
+++ b/hbase-native-client/bin/copy-protobuf.sh
@@ -1,4 +1,20 @@
 #!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 set -euo pipefail
 IFS=$'\n\t'
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/format-code.sh
--
diff --git a/hbase-native-client/bin/format-code.sh 
b/hbase-native-client/bin/format-code.sh
index 24d6515..36a114d 100755
--- a/hbase-native-client/bin/format-code.sh
+++ b/hbase-native-client/bin/format-code.sh
@@ -1,4 +1,20 @@
 #!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 set -euo pipefail
 IFS=$'\n\t'
 

[33/50] [abbrv] hbase git commit: HBASE-15687 Allow decoding more than GetResponse from the server

[eclark]

HBASE-15687 Allow decoding more than GetResponse from the server

Summary:
We'll need more than get's for the client to be usable. So now the
Request class contains the protobufs needed to encode and decode
rpc's.

I also added some helper methods to create initial requests.

Test Plan: It compiles and still gets data from HBase meta

Differential Revision: https://reviews.facebook.net/D57327


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/14a33d7d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/14a33d7d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/14a33d7d

Branch: refs/heads/HBASE-14850
Commit: 14a33d7dbc7c9118f83be24925d1ff5c603cc352
Parents: a1069eb
Author: Elliott Clark 
Authored: Wed Apr 27 15:27:09 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/connection/BUCK |  1 +
 .../connection/client-dispatcher.cc |  6 +--
 .../connection/client-dispatcher.h  |  6 +--
 .../connection/client-handler.cc| 30 ++---
 hbase-native-client/connection/client-handler.h | 18 ++--
 .../connection/connection-factory.cc|  6 +--
 .../connection/connection-factory.h |  2 +-
 hbase-native-client/connection/pipeline.h   |  3 +-
 hbase-native-client/connection/request.cc   | 45 
 hbase-native-client/connection/request.h| 25 +--
 hbase-native-client/core/simple-client.cc   | 13 +++---
 11 files changed, 123 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/BUCK
--
diff --git a/hbase-native-client/connection/BUCK 
b/hbase-native-client/connection/BUCK
index 5067708..d393885 100644
--- a/hbase-native-client/connection/BUCK
+++ b/hbase-native-client/connection/BUCK
@@ -32,6 +32,7 @@ cxx_library(name="connection",
 "client-handler.cc",
 "connection-factory.cc",
 "pipeline.cc",
+"request.cc",
 ],
 deps=[
 "//if:if",

http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-dispatcher.cc
--
diff --git a/hbase-native-client/connection/client-dispatcher.cc 
b/hbase-native-client/connection/client-dispatcher.cc
index 25cff7d..eea0a17 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -35,16 +35,16 @@ void ClientDispatcher::read(Context *ctx, Response in) {
   p.setValue(in);
 }
 
-Future ClientDispatcher::operator()(Request arg) {
+Future ClientDispatcher::operator()(std::unique_ptr arg) {
   auto call_id = ++current_call_id_;
 
-  arg.set_call_id(call_id);
+  arg->set_call_id(call_id);
   auto  = requests_[call_id];
   auto f = p.getFuture();
   p.setInterruptHandler([call_id, this](const folly::exception_wrapper ) {
 this->requests_.erase(call_id);
   });
-  this->pipeline_->write(arg);
+  this->pipeline_->write(std::move(arg));
 
   return f;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
index 89c7119..877e877 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -27,11 +27,11 @@
 
 namespace hbase {
 class ClientDispatcher
-: public wangle::ClientDispatcherBase {
+: public wangle::ClientDispatcherBase {
 public:
   void read(Context *ctx, Response in) override;
-  folly::Future operator()(Request arg) override;
+  folly::Future operator()(std::unique_ptr arg) override;
   folly::Future close(Context *ctx) override;
   folly::Future close() override;
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 205993a7..abcf5c1 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -47,14 +47,30 @@ void ClientHandler::read(Context *ctx, 
std::unique_ptr buf) {
   

[43/50] [abbrv] hbase git commit: HBASE-15731 Add on a connection pool

[eclark]

HBASE-15731 Add on a connection pool

Summary:
Add on a connection pool protected by read write mutex.
Add on a service filter that will remove a connection from a connection pool 
when closed

Test Plan: Need to add on tests.

Differential Revision: https://reviews.facebook.net/D57411


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/baf863f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/baf863f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/baf863f6

Branch: refs/heads/HBASE-14850
Commit: baf863f6829d5d68f706e7b49b360eb1d2927708
Parents: c1140d7
Author: Elliott Clark 
Authored: Thu Apr 28 19:53:42 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/connection/BUCK |   5 +
 .../connection/connection-factory.cc|   4 +-
 .../connection/connection-factory.h |   5 +-
 .../connection/connection-pool-test.cc  |  77 ++
 .../connection/connection-pool.cc   |  89 
 .../connection/connection-pool.h|  59 +++
 hbase-native-client/connection/service.h|   4 +-
 hbase-native-client/core/BUCK   |   4 -
 hbase-native-client/core/client.h   |   3 -
 hbase-native-client/core/get-request.cc |  19 
 hbase-native-client/core/get-request.h  |  35 ---
 hbase-native-client/core/get-result.cc  |  19 
 hbase-native-client/core/get-result.h   |  32 --
 hbase-native-client/core/location-cache.cc  |   1 -
 hbase-native-client/core/simple-client.cc   | 105 +++
 15 files changed, 297 insertions(+), 164 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/BUCK
--
diff --git a/hbase-native-client/connection/BUCK 
b/hbase-native-client/connection/BUCK
index d393885..96f2136 100644
--- a/hbase-native-client/connection/BUCK
+++ b/hbase-native-client/connection/BUCK
@@ -22,6 +22,7 @@ cxx_library(name="connection",
 "client-dispatcher.h",
 "client-handler.h",
 "connection-factory.h",
+"connection-pool.h",
 "pipeline.h",
 "request.h",
 "response.h",
@@ -31,6 +32,7 @@ cxx_library(name="connection",
 "client-dispatcher.cc",
 "client-handler.cc",
 "connection-factory.cc",
+"connection-pool.cc",
 "pipeline.cc",
 "request.cc",
 ],
@@ -42,3 +44,6 @@ cxx_library(name="connection",
 "//third-party:wangle",
 ],
 visibility=['//core/...', ], )
+cxx_test(name="connection-pool-test",
+ srcs=["connection-pool-test.cc", ],
+ deps=[":connection", ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-factory.cc
--
diff --git a/hbase-native-client/connection/connection-factory.cc 
b/hbase-native-client/connection/connection-factory.cc
index 7073f9d..b546269 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -45,8 +45,8 @@ ConnectionFactory::ConnectionFactory() {
   bootstrap_.pipelineFactory(std::make_shared());
 }
 
-std::shared_ptr>
-ConnectionFactory::make_connection(std::string host, int port) {
+std::shared_ptr
+ConnectionFactory::make_connection(const std::string , int port) {
   // Connect to a given server
   // Then when connected create a ClientDispactcher.
   auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();

http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-factory.h
--
diff --git a/hbase-native-client/connection/connection-factory.h 
b/hbase-native-client/connection/connection-factory.h
index 8d1d2f0..5a45316 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -31,8 +31,9 @@ namespace hbase {
 class ConnectionFactory {
 public:
   ConnectionFactory();
-  std::shared_ptr>
-  make_connection(std::string host, int port);
+
+  virtual std::shared_ptr make_connection(const std::string 
,
+int port);
 
 private:
   wangle::ClientBootstrap bootstrap_;

[41/50] [abbrv] hbase git commit: HBASE-15690 Add utility to get current username

[eclark]

HBASE-15690 Add utility to get current username

Summary:
Add a class to fine the username of the current process.
It will only call out once and is multithread safe

Test Plan: Unit Test

Differential Revision: https://reviews.facebook.net/D57081


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b2d65f1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b2d65f1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b2d65f1

Branch: refs/heads/HBASE-14850
Commit: 8b2d65f1aa7c7dddcde4be03133f8c35e1fbcba2
Parents: b23e863
Author: Elliott Clark 
Authored: Thu Apr 21 21:17:11 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/utils/BUCK  | 40 +
 hbase-native-client/utils/user-util-test.cc | 34 +++
 hbase-native-client/utils/user-util.cc  | 55 
 hbase-native-client/utils/user-util.h   | 37 
 4 files changed, 166 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/BUCK
--
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
new file mode 100644
index 000..2b65b12
--- /dev/null
+++ b/hbase-native-client/utils/BUCK
@@ -0,0 +1,40 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="utils",
+exported_headers=[
+"user-util.h",
+],
+srcs=[
+"user-util.cc",
+],
+deps=[
+  '//third-party:folly',
+],
+visibility=[
+'PUBLIC',
+],
+tests=[
+  ":user-util-test"
+],)
+cxx_test(name="user-util-test",
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)

http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util-test.cc
--
diff --git a/hbase-native-client/utils/user-util-test.cc 
b/hbase-native-client/utils/user-util-test.cc
new file mode 100644
index 000..2a7434f
--- /dev/null
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include 
+#include 
+#include 
+
+#include "utils/user-util.h"
+
+using namespace std;
+using namespace hbase;
+
+TEST(TestUserUtil, TestGetSomething) {
+  UserUtil u_util;
+  string name = u_util.user_name();
+
+  ASSERT_GT(name.length(), 0);
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util.cc
--
diff --git a/hbase-native-client/utils/user-util.cc 
b/hbase-native-client/utils/user-util.cc
new file mode 100644
index 000..c4427e3
--- /dev/null
+++ b/hbase-native-client/utils/user-util.cc
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * 

[40/50] [abbrv] hbase git commit: HBASE-15655 Set up doxygen for documentation

[eclark]

HBASE-15655 Set up doxygen for documentation

Summary:
Add on doxygen config.
It outputs to the docs folder.
Added on a makefile

Test Plan:
doxygen hbase.doxygen

Differential Revision: https://reviews.facebook.net/D57663


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f034294f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f034294f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f034294f

Branch: refs/heads/HBASE-14850
Commit: f034294f3a72bcb6c11330d6fa1f9b6c028f1f1c
Parents: 190fc7a
Author: Elliott Clark 
Authored: Wed May 4 18:19:33 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .gitignore|   1 +
 hbase-native-client/Dockerfile|   2 +-
 hbase-native-client/Makefile  |  20 +++
 hbase-native-client/hbase.doxygen | 319 +
 4 files changed, 341 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/.gitignore
--
diff --git a/.gitignore b/.gitignore
index 626a4d0..1b1e9d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ hbase-*/test
 *.ipr
 patchprocess/
 dependency-reduced-pom.xml
+docs/

http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 8e33e74..be0cbc6 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -24,7 +24,7 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g 
-fno-omit-frame-pointer -O2 -p
 
 ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
 
-RUN apt-get install -y vim maven inetutils-ping python-pip && \
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && 
\
   pip install yapf && \
   apt-get -qq clean && \
   apt-get -y -qq autoremove && \

http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
new file mode 100644
index 000..90ec6ea
--- /dev/null
+++ b/hbase-native-client/Makefile
@@ -0,0 +1,20 @@
+build:
+   $(shell buck build core/... )
+
+check:
+   $(shell buck test --all --no-results-cache )
+
+doc:
+   $(shell doxygen hbase.doxygen > /dev/null )
+
+clean:
+   $(shell rm -rf docs )
+
+help:
+   @echo "Available targets:"
+   @echo ""
+   @echo " build : will build everything."
+   @echo " clean : will remove the docs folder"
+   @echo " check : will test everything."
+
+all: build doc check

http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/hbase.doxygen
--
diff --git a/hbase-native-client/hbase.doxygen 
b/hbase-native-client/hbase.doxygen
new file mode 100644
index 000..22e2d21
--- /dev/null
+++ b/hbase-native-client/hbase.doxygen
@@ -0,0 +1,319 @@
+# Doxyfile 1.8.11
+
+#---
+# Project related configuration options
+#---
+DOXYFILE_ENCODING  = UTF-8
+PROJECT_NAME   = "HBase Native Client"
+PROJECT_NUMBER =
+PROJECT_BRIEF  =
+PROJECT_LOGO   = 
../hbase-server/src/main/resources/hbase-webapps/static/hbase_logo_small.png
+OUTPUT_DIRECTORY   = docs
+CREATE_SUBDIRS = YES
+ALLOW_UNICODE_NAMES= NO
+OUTPUT_LANGUAGE= English
+BRIEF_MEMBER_DESC  = YES
+REPEAT_BRIEF   = YES
+ABBREVIATE_BRIEF   =
+ALWAYS_DETAILED_SEC= NO
+INLINE_INHERITED_MEMB  = NO
+FULL_PATH_NAMES= NO
+STRIP_FROM_PATH=
+STRIP_FROM_INC_PATH=
+SHORT_NAMES= NO
+JAVADOC_AUTOBRIEF  = NO
+QT_AUTOBRIEF   = NO
+MULTILINE_CPP_IS_BRIEF = NO
+INHERIT_DOCS   = YES
+SEPARATE_MEMBER_PAGES  = NO
+TAB_SIZE   = 4
+ALIASES=
+TCL_SUBST  =
+OPTIMIZE_OUTPUT_FOR_C  = NO
+OPTIMIZE_OUTPUT_JAVA   = NO
+OPTIMIZE_FOR_FORTRAN   = NO
+OPTIMIZE_OUTPUT_VHDL   = NO
+EXTENSION_MAPPING  =
+MARKDOWN_SUPPORT   = YES
+AUTOLINK_SUPPORT   = YES
+BUILTIN_STL_SUPPORT= NO
+CPP_CLI_SUPPORT= NO
+SIP_SUPPORT= NO
+IDL_PROPERTY_SUPPORT   = YES
+DISTRIBUTE_GROUP_DOC   = NO
+GROUP_NESTED_COMPOUNDS = NO
+SUBGROUPING= YES
+INLINE_GROUPED_CLASSES = NO
+INLINE_SIMPLE_STRUCTS  = NO
+TYPEDEF_HIDES_STRUCT   = NO
+LOOKUP_CACHE_SIZE  = 0

[26/50] [abbrv] hbase git commit: HBASE-14855 Connect to regionserver

[eclark]

HBASE-14855 Connect to regionserver

Summary:
* Client dispatcher to put request and response together with call id. Though 
right now this is a pretty empty version of that.
* pipeline. The Serialization pipeline.
* client-serilization-handler.h This does the work of sending the preamable, 
the connection header, and length prepending data. This will obviously need to 
be split up into h and cc files

Test Plan: Use simple-client to connect to a standalone server

Differential Revision: https://reviews.facebook.net/D56385


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b23e8630
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b23e8630
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b23e8630

Branch: refs/heads/HBASE-14850
Commit: b23e8630cfe9148c858c4860e64e1693db0d8bd5
Parents: bbeaacb
Author: Elliott Clark 
Authored: Mon Apr 4 13:52:05 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/core/BUCK   |  30 +-
 hbase-native-client/core/client-dispatcher.cc   |  54 ++
 hbase-native-client/core/client-dispatcher.h|  42 
 .../core/client-serialize-handler.cc| 104 ++
 .../core/client-serialize-handler.h |  44 
 hbase-native-client/core/client.cc  |  16 ++-
 hbase-native-client/core/client.h   |  18 +++-
 hbase-native-client/core/connection-factory.cc  |  57 ++
 hbase-native-client/core/connection-factory.h   |  41 
 hbase-native-client/core/get-request.cc |  19 
 hbase-native-client/core/get-request.h  |  35 +++
 hbase-native-client/core/get-result.cc  |  19 
 hbase-native-client/core/get-result.h   |  32 ++
 hbase-native-client/core/location-cache-test.cc |  25 -
 hbase-native-client/core/location-cache.cc  |  62 +--
 hbase-native-client/core/location-cache.h   |  29 -
 .../core/native-client-test-env.cc  |   6 +-
 hbase-native-client/core/pipeline.cc|  42 
 hbase-native-client/core/pipeline.h |  34 ++
 hbase-native-client/core/request.h  |  33 ++
 hbase-native-client/core/response.h |  34 ++
 hbase-native-client/core/service.h  |  26 +
 hbase-native-client/core/simple-client.cc   |  57 ++
 .../core/simple-native-client-test.cc   |   5 +-
 hbase-native-client/core/table-name.cc  |  19 
 hbase-native-client/core/table-name.h   |  32 ++
 hbase-native-client/if/BUCK |  24 -
 hbase-native-client/third-party/BUCK| 105 +--
 28 files changed, 948 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index d1e89d1..2b00d66 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -16,35 +16,52 @@
 # limitations under the License.
 
 cxx_library(name="core",
-headers=[
+exported_headers=[
 "admin.h",
+"client-dispatcher.h",
+"client-serialize-handler.h",
 "client.h",
+"connection-factory.h",
 "connection.h",
 "connection_attr.h",
 "delete.h",
+"get-request.h",
+"get-result.h",
 "get.h",
 "hbase_macros.h",
+"location-cache.h",
 "mutation.h",
+"pipeline.h",
 "put.h",
+"request.h",
+"response.h",
 "scanner.h",
-"location-cache.h",
+"service.h",
+"table-name.h",
 ],
 srcs=[
 "admin.cc",
+"client-dispatcher.cc",
+"client-serialize-handler.cc",
 "client.cc",
+"connection-factory.cc",
 "connection.cc",
+"delete.cc",
+"get-request.cc",
+"get-result.cc",
 "get.cc",
+"location-cache.cc",
 "mutation.cc",
+"pipeline.cc",
 "put.cc",
-"delete.cc",
 "scanner.cc",
-"location-cache.cc",
+"table-name.cc",
 ],
 deps=[
 "//if:if",
-"//third-party:zookeeper_mt",
 

[39/50] [abbrv] hbase git commit: HBASE-15777 Fix needs header in client handler

[eclark]

HBASE-15777 Fix needs header in client handler


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/16e1de78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/16e1de78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/16e1de78

Branch: refs/heads/HBASE-14850
Commit: 16e1de78a6a0fa92adbd330890f213d8763adb83
Parents: 79b5085
Author: Elliott Clark 
Authored: Thu May 5 13:51:18 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-handler.cc| 31 
 hbase-native-client/connection/client-handler.h | 19 +++-
 2 files changed, 36 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/16e1de78/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index b92ad89..4fdb7ae 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
 using google::protobuf::Message;
 
 ClientHandler::ClientHandler(std::string user_name)
-: user_name_(user_name), need_send_header_(true), serde_(),
+: user_name_(user_name), serde_(), header_info_(),
   resp_msgs_(
   make_unique>(5000)) {}
@@ -81,22 +81,27 @@ void ClientHandler::read(Context *ctx, 
std::unique_ptr buf) {
   }
 }
 
-// TODO(eclark): Figure out how to handle the
-// network errors that are going to come.
 Future ClientHandler::write(Context *ctx, std::unique_ptr r) {
   // Keep track of if we have sent the header.
-  if (UNLIKELY(need_send_header_)) {
-need_send_header_ = false;
+  //
+  // even though the bool is atomic we can load it lazily here.
+  if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
 
-// Should we be sending just one fireWrite?
-// Right now we're sending one for the header
-// and one for the request.
+// Grab the lock.
+// We need to make sure that no one gets past here without there being a
+// hearder sent.
+std::lock_guard lock(header_info_->mutex_);
+
+// Now see if we are the first thread to get through.
 //
-// That doesn't seem like too bad, but who knows.
-auto pre = serde_.Preamble();
-auto header = serde_.Header(user_name_);
-pre->appendChain(std::move(header));
-ctx->fireWrite(std::move(pre));
+// If this is the first thread to get through then the
+// need_send_header will have been true before this.
+if (header_info_->need_.exchange(false)) {
+  auto pre = serde_.Preamble();
+  auto header = serde_.Header(user_name_);
+  pre->appendChain(std::move(header));
+  ctx->fireWrite(std::move(pre));
+}
   }
 
   resp_msgs_->insert(r->call_id(), r->resp_msg());

http://git-wip-us.apache.org/repos/asf/hbase/blob/16e1de78/hbase-native-client/connection/client-handler.h
--
diff --git a/hbase-native-client/connection/client-handler.h 
b/hbase-native-client/connection/client-handler.h
index be5143c..1a4275f 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -21,6 +21,8 @@
 #include 
 #include 
 
+#include 
+#include 
 #include 
 
 #include "serde/rpc.h"
@@ -29,6 +31,7 @@
 namespace hbase {
 class Request;
 class Response;
+class HeaderInfo;
 }
 namespace google {
 namespace protobuf {
@@ -37,6 +40,7 @@ class Message;
 }
 
 namespace hbase {
+
 class ClientHandler : public wangle::Handler {
@@ -47,7 +51,7 @@ public:
std::unique_ptr r) override;
 
 private:
-  bool need_send_header_;
+  std::unique_ptr header_info_;
   std::string user_name_;
   RpcSerde serde_;
 
@@ -56,4 +60,17 @@ private:
   uint32_t, std::shared_ptr>>
   resp_msgs_;
 };
+
+/**
+ * Class to contain the info about if the connection header and preamble has
+ * been sent.
+ *
+ * We use a serperate class here so that ClientHandler is relocatable.
+ */
+class HeaderInfo {
+public:
+  HeaderInfo() : need_(true), mutex_() {}
+  std::atomic need_;
+  std::mutex mutex_;
+};
 } // namespace hbase

[04/50] [abbrv] hbase git commit: HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove TimeRangeTracker as point of contention when many threads reading a StoreFile Fixes HBASE-16074 ITBLL fails,

[eclark]

HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove
 TimeRangeTracker as point of contention when many threads reading a StoreFile
 Fixes HBASE-16074 ITBLL fails, reports lost big or tiny families broken
 scanning because of a side effect of a clean up in  HBASE-15650 to make
 TimeRange construction consistent exposed a latent issue in
 TimeRange#compare. See HBASE-16074 for more detail.

Also change HFile Writer constructor so we pass in the TimeRangeTracker, if one,
on construction rather than set later (the flag and reference were not volatile
so could have made for issues in concurrent case). And make sure the 
construction
of a TimeRange from a TimeRangeTracer on open of an HFile Reader never makes a
bad minimum value, one that would preclude us reading any values from a file
(set min to 0)

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
 Call through to next constructor (if minStamp was 0, we'd skip setting
 allTime=true). Add asserts that timestamps are not < 0 cos it messes
 us up if they are (we already were checking for < 0 on construction but
 assert passed in timestamps are not < 0).

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
 Add constructor override that takes a TimeRangeTracker (set when flushing
 but not when compacting)

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
 Add override creating an HFile in tmp that takes a TimeRangeTracker

M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
 Add override for HFile Writer that takes a TimeRangeTracker Take it on
 construction instead of having it passed by a setter later (flags and
 reference set by the setter were not volatile... could have been prob
 in concurrent case)

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
 Log WARN if bad initial TimeRange value (and then 'fix' it)

M 
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
 A few tests to prove serialization works as expected and that we'll get a bad 
min if not constructed properly.

M 
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
 Handle OLDEST_TIMESTAMP explictly. Don't expect TimeRange to do it.

M 
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
 Refactor from junit3 to junit4 and add test for this weird case.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/496fd983
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/496fd983
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/496fd983

Branch: refs/heads/HBASE-14850
Commit: 496fd9837a0fb199a516758a632fecfe59b0b480
Parents: 3c39cbd
Author: stack 
Authored: Fri Jul 8 18:18:01 2016 -0700
Committer: stack 
Committed: Fri Jul 8 18:20:31 2016 -0700

--
 .../org/apache/hadoop/hbase/client/Query.java   |  4 +-
 .../org/apache/hadoop/hbase/HConstants.java |  5 +
 .../org/apache/hadoop/hbase/KeyValueUtil.java   |  2 +-
 .../org/apache/hadoop/hbase/io/TimeRange.java   | 99 
 .../hbase/mob/DefaultMobStoreFlusher.java   |  5 +-
 .../hbase/regionserver/DefaultStoreFlusher.java |  4 +-
 .../regionserver/ExplicitColumnTracker.java |  4 +-
 .../hadoop/hbase/regionserver/HStore.java   | 27 +-
 .../hbase/regionserver/ScanQueryMatcher.java| 11 ++-
 .../apache/hadoop/hbase/regionserver/Store.java | 20 +++-
 .../hadoop/hbase/regionserver/StoreFile.java|  1 -
 .../hbase/regionserver/StoreFileReader.java |  2 +-
 .../hbase/regionserver/StoreFileWriter.java | 61 
 .../hadoop/hbase/regionserver/StoreScanner.java |  3 +-
 .../hbase/regionserver/StripeStoreFlusher.java  |  4 +-
 .../hbase/regionserver/TimeRangeTracker.java| 23 ++---
 .../hbase/regionserver/TestQueryMatcher.java| 35 +--
 .../regionserver/TestTimeRangeTracker.java  | 40 +++-
 18 files changed, 250 insertions(+), 100 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
index 53062a0..53dd2c1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
@@ -203,6 +203,4 @@ public abstract class Query extends OperationWithAttributes 
{
   public Map getColumnFamilyTimeRange() {
 return this.colFamTimeRangeMap;
   }
-
-
-}
+}
\ No newline at end of file

[34/50] [abbrv] hbase git commit: HBASE-15792 Add on a test for locating a region

[eclark]

HBASE-15792 Add on a test for locating a region

Summary:
Add on a TestUtil class that will start a cluster.
Add on the ability to run a shell command in a cluster.
Throw an exception if the table isn't the correct table when looking up a 
region.
Add a test to test that getting a region throws when the table doesn't exist.
Add a test to make sure that getting a region location works when the table 
does exist.

Test Plan:
Added unit tests.
Changed unit tests.

Differential Revision: https://reviews.facebook.net/D57891


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eb4cde49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eb4cde49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eb4cde49

Branch: refs/heads/HBASE-14850
Commit: eb4cde498f0fed8fd1e7ff35b71c69cabe09768d
Parents: d3a6efd
Author: Elliott Clark 
Authored: Mon May 9 12:06:54 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/start-local-hbase.sh|  5 +-
 .../connection/client-handler.cc|  3 +-
 hbase-native-client/core/BUCK   |  4 +-
 hbase-native-client/core/client.cc  |  1 +
 hbase-native-client/core/location-cache-test.cc | 29 ++-
 hbase-native-client/core/location-cache.cc  | 10 +++-
 hbase-native-client/core/location-cache.h   | 10 +++-
 hbase-native-client/core/test-env.cc| 45 
 hbase-native-client/test-util/BUCK  | 29 +++
 hbase-native-client/test-util/test-util.cc  | 54 
 hbase-native-client/test-util/test-util.h   | 51 ++
 11 files changed, 187 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/bin/start-local-hbase.sh
--
diff --git a/hbase-native-client/bin/start-local-hbase.sh 
b/hbase-native-client/bin/start-local-hbase.sh
index cfc71f9..49637d6 100755
--- a/hbase-native-client/bin/start-local-hbase.sh
+++ b/hbase-native-client/bin/start-local-hbase.sh
@@ -21,7 +21,8 @@
 rm -rf /tmp/hbase-*
 
 # Start the master/regionservers.
-$PWD/../bin/start-hbase.sh
+T_DIR=${1:-"/tmp/hbase-testing"}
+$PWD/../bin/start-hbase.sh -Dhbase.tmp.dir="${T_DIR}"
 
 until [ $(curl -s -o /dev/null -I -w "%{http_code}" 
http://localhost:16010/jmx) == "200" ]
 do
@@ -30,4 +31,4 @@ do
 done
 
 # This sucks, but master can easily be up and meta not be assigned yet.
-sleep 30
+sleep 10

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 2e3fcd3..3409b80 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,8 @@ using hbase::pb::GetResponse;
 using google::protobuf::Message;
 
 ClientHandler::ClientHandler(std::string user_name)
-: user_name_(user_name), serde_(), 
once_flag_(std::make_unique()),
+: user_name_(user_name), serde_(),
+  once_flag_(std::make_unique()),
   resp_msgs_(
   make_unique>(5000)) {}

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 485f9ba..1c926e3 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -46,10 +46,10 @@ cxx_library(
 ], )
 cxx_test(name="location-cache-test",
  srcs=[
- "test-env.cc",
  "location-cache-test.cc",
  ],
- deps=[":core", ],
+ deps=[":core",
+   "//test-util:test-util", ],
  run_test_separately=True, )
 cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/client.cc
--
diff --git a/hbase-native-client/core/client.cc 
b/hbase-native-client/core/client.cc
index 1e80998..82ecd22 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -22,6 +22,7 @@
 #include 
 
 #include 
+#include 
 
 using namespace folly;
 using namespace std;

http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/location-cache-test.cc
--
diff --git 

[46/50] [abbrv] hbase git commit: HBASE-15771 Document all the public classes

[eclark]

HBASE-15771 Document all the public classes

Summary: Add on a bunch of documentation around implementation so far.

Test Plan:
make doc
Doxygen has no warnings

Differential Revision: https://reviews.facebook.net/D57753


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fdf00239
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fdf00239
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fdf00239

Branch: refs/heads/HBASE-14850
Commit: fdf00239c691d0de9b1b97745dafe1cea57dbf9c
Parents: 16e1de7
Author: Elliott Clark 
Authored: Thu May 5 13:14:20 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/BUILDING.md | 18 +-
 .../connection/client-dispatcher.h  | 11 +++-
 .../connection/client-handler.cc| 37 +---
 hbase-native-client/connection/client-handler.h | 36 +++-
 .../connection/connection-factory.h | 18 ++
 .../connection/connection-pool.cc   |  5 +-
 .../connection/connection-pool.h| 44 +-
 hbase-native-client/connection/pipeline.h   | 18 ++
 hbase-native-client/connection/request.h| 32 ++
 hbase-native-client/connection/response.h   | 33 +--
 hbase-native-client/core/BUCK   |  1 -
 hbase-native-client/core/client.h   | 12 
 hbase-native-client/core/connection.cc  | 20 ---
 hbase-native-client/core/connection.h   | 26 -
 hbase-native-client/core/location-cache.cc  | 14 -
 hbase-native-client/core/location-cache.h   | 36 ++--
 hbase-native-client/core/meta-utils.h   | 12 
 hbase-native-client/core/put.cc | 21 ---
 hbase-native-client/core/put.h  | 27 -
 hbase-native-client/core/region-location.h  | 41 +
 hbase-native-client/serde/rpc.cc|  1 -
 hbase-native-client/serde/rpc.h | 61 +++-
 hbase-native-client/serde/zk.h  | 14 +
 hbase-native-client/utils/user-util.h   | 18 ++
 24 files changed, 387 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/BUILDING.md
--
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
index 20ef2a0..4c06776 100644
--- a/hbase-native-client/BUILDING.md
+++ b/hbase-native-client/BUILDING.md
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
 under the License.
 -->
 
-#Building HBase native client
+# Building HBase native client
 
 The HBase native client build using buck and produces a linux library.
 
@@ -58,6 +58,20 @@ buck build //core:simple-client
 ```
 
 That will build the library, then build and test everything, then build
-the simple-client binary. Buck will find all modules used, and compile 
+the simple-client binary. Buck will find all modules used, and compile
 them in parallel, caching the results. Output from buck is in the buck-out
 foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
+
+
+# Make
+
+If learning buck isn't your thing there is a Makefile wrapper for your
+convenience.
+
+```
+make help
+make check
+make clean
+make all
+make build
+```

http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
index 4435a1b..4bfb35d 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -30,17 +30,22 @@
 #include "connection/response.h"
 
 namespace hbase {
+/**
+ * Dispatcher that assigns a call_id and then routes the response back to the 
future.
+ */
 class ClientDispatcher
 : public wangle::ClientDispatcherBase {
 public:
+  /** Create a new ClientDispatcher */
   ClientDispatcher();
-  ~ClientDispatcher() {
-LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
-  }
+  /** Read a response off the pipeline. */
   void read(Context *ctx, Response in) override;
+  /** Take a request as a call and send it down the pipeline. */
   folly::Future operator()(std::unique_ptr arg) override;
+  /** Close the dispatcher and the associated pipeline. */
   folly::Future close(Context *ctx) override;
+  /** Close the dispatcher and the associated pipeline. */
   folly::Future 

[22/50] [abbrv] hbase git commit: HBASE-14854 Read meta location from zk

[eclark]

HBASE-14854 Read meta location from zk


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/26ba4ed4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/26ba4ed4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/26ba4ed4

Branch: refs/heads/HBASE-14850
Commit: 26ba4ed461600a0bc8581d2ba9ba4230b71e992b
Parents: 27ee8ba
Author: Elliott Clark 
Authored: Sat Mar 5 00:09:08 2016 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile  |  20 ++-
 hbase-native-client/bin/start-docker.sh |   5 +-
 hbase-native-client/core/BUCK   | 106 +++---
 .../core/HBaseNativeClientTestEnv.cc|  42 --
 .../core/SampleNativeClientTest.cc  |  28 
 hbase-native-client/core/location-cache-test.cc |  14 ++
 hbase-native-client/core/location-cache.cc  |  67 +
 hbase-native-client/core/location-cache.h   |  35 +
 .../core/native-client-test-env.cc  |  42 ++
 .../core/simple-native-client-test.cc   |  28 
 hbase-native-client/core/test_env.h |   2 +
 hbase-native-client/if/BUCK |   1 +
 hbase-native-client/third-party/BUCK| 138 ++-
 13 files changed, 339 insertions(+), 189 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1364d22..36959a5 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM cpp_update 
+FROM pjameson/buck-folly-watchman
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5
@@ -25,20 +25,26 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g 
-fno-omit-frame-pointer -O3 -p
 RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
 RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git 
/usr/src/protobuf && \
   cd /usr/src/protobuf/ && \
+  ldconfig && \
   ./autogen.sh && \
-  ./configure --disable-shared && \
+  ./configure && \
   make && \
-  make check && \
-  make install
+  make install && \ 
+  make clean && \
+  rm -rf .git
+
 RUN cd /usr/src && \
   wget 
http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz 
&& \ 
   tar zxf zookeeper-3.4.8.tar.gz && \ 
   rm -rf zookeeper-3.4.8.tar.gz && \
   cd zookeeper-3.4.8 && \
   cd src/c && \
-  ./configure --disable-shared && \
+  ldconfig && \
+  ./configure && \
   make && \
   make install && \
-  make clean 
+  make clean
+
+RUN ldconfig
 
-WORKDIR /usr/local/src/hbase/hbase-native-client
+WORKDIR /usr/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 4426705..725ed6a 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -19,8 +19,11 @@
 set -e
 set -x
 
+# Try out some standard docker machine names that could work
 eval "$(docker-machine env docker-vm)"
 eval "$(docker-machine env dinghy)"
+
+# Build the image
 docker build -t hbase_native .
 
 
@@ -36,6 +39,6 @@ fi;
 
 docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-   -v ${PWD}/..:/usr/local/src/hbase \
+   -v ${PWD}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash

http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index ef027a1..817b5a0 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -15,52 +15,62 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+cxx_library(name="core",
+headers=[
+"admin.h",
+"client.h",
+"connection.h",
+"connection_attr.h",
+"delete.h",
+"get.h",
+"hbase_macros.h",
+"mutation.h",
+"put.h",
+"scanner.h",
+"location-cache.h",
+],
+srcs=[
+"admin.cc",
+"client.cc",
+

[20/50] [abbrv] hbase git commit: HBASE-14853 Add on protobuf to c++ chain

[eclark]

HBASE-14853 Add on protobuf to c++ chain


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e00e3a04
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e00e3a04
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e00e3a04

Branch: refs/heads/HBASE-14850
Commit: e00e3a046dddf46cef62d1e5ff3c2dda10b9e559
Parents: 543
Author: Elliott Clark 
Authored: Mon Dec 28 15:33:52 2015 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/start-docker.sh |   1 +
 hbase-native-client/core/BUCK   |   1 +
 hbase-native-client/core/client.cc  |   4 +
 hbase-native-client/core/client.h   |   2 +
 hbase-native-client/if/AccessControl.proto  | 123 +++
 hbase-native-client/if/Admin.proto  | 309 
 hbase-native-client/if/Aggregate.proto  |  63 ++
 hbase-native-client/if/Authentication.proto |  82 ++
 hbase-native-client/if/BUCK |  36 +
 hbase-native-client/if/Cell.proto   |  67 ++
 hbase-native-client/if/Client.proto | 472 +++
 hbase-native-client/if/ClusterId.proto  |  34 +
 hbase-native-client/if/ClusterStatus.proto  | 224 ++
 hbase-native-client/if/Comparator.proto |  74 ++
 hbase-native-client/if/Encryption.proto |  33 +
 hbase-native-client/if/ErrorHandling.proto  |  58 ++
 hbase-native-client/if/FS.proto |  45 ++
 hbase-native-client/if/Filter.proto | 170 
 hbase-native-client/if/HBase.proto  | 258 ++
 hbase-native-client/if/HFile.proto  |  49 ++
 hbase-native-client/if/LoadBalancer.proto   |  29 +
 hbase-native-client/if/MapReduce.proto  |  37 +
 hbase-native-client/if/Master.proto | 778 +++
 hbase-native-client/if/MasterProcedure.proto| 245 ++
 hbase-native-client/if/MultiRowMutation.proto   |  45 ++
 hbase-native-client/if/Procedure.proto  | 119 +++
 hbase-native-client/if/Quota.proto  |  76 ++
 hbase-native-client/if/RPC.proto| 136 
 hbase-native-client/if/RegionNormalizer.proto   |  28 +
 hbase-native-client/if/RegionServerStatus.proto | 158 
 hbase-native-client/if/RowProcessor.proto   |  45 ++
 hbase-native-client/if/SecureBulkLoad.proto |  72 ++
 hbase-native-client/if/Snapshot.proto   |  66 ++
 hbase-native-client/if/Tracing.proto|  33 +
 hbase-native-client/if/VisibilityLabels.proto   |  83 ++
 hbase-native-client/if/WAL.proto| 172 
 hbase-native-client/if/ZooKeeper.proto  | 155 
 hbase-native-client/rpc/CMakeLists.txt  |  17 -
 hbase-native-client/third-party/BUCK|   4 +-
 39 files changed, 4385 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index bf38912..4426705 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -20,6 +20,7 @@ set -e
 set -x
 
 eval "$(docker-machine env docker-vm)"
+eval "$(docker-machine env dinghy)"
 docker build -t hbase_native .
 
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 2e4e755..ef027a1 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -41,6 +41,7 @@ cxx_binary(
   "scanner.cc",
],
deps = [
+   "//if:if",
"//third-party:folly",
"//third-party:wangle",
],

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/client.cc
--
diff --git a/hbase-native-client/core/client.cc 
b/hbase-native-client/core/client.cc
index 98cf38a..a04daee 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -24,9 +24,13 @@
 #include 
 #include 
 
+#include "if/ZooKeeper.pb.h"
+
 using namespace folly;
+using namespace hbase::pb;
 
 int main(int argc, char *argv[]) {
+  MetaRegionServer mrs;
   google::ParseCommandLineFlags(, , true);
   google::InitGoogleLogging(argv[0]);
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/client.h
--
diff --git 

[49/50] [abbrv] hbase git commit: HBASE-15724 Use explicit docker image

[eclark]

HBASE-15724 Use explicit docker image

Summary:
Set the tag for cpp docker image
Don't set the env on command line

Test Plan: Build it.

Differential Revision: https://reviews.facebook.net/D57315


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1069eb1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1069eb1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1069eb1

Branch: refs/heads/HBASE-14850
Commit: a1069eb13978213027d34a09e4a15f3fdea83f5d
Parents: 2aaadab
Author: Elliott Clark 
Authored: Wed Apr 27 10:51:05 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile  | 4 +++-
 hbase-native-client/bin/start-docker.sh | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a1069eb1/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1524c56..8e33e74 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,13 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman
+FROM pjameson/buck-folly-watchman:20160425
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5
 ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 
-pthread"
 ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 
-pthread"
 
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+
 RUN apt-get install -y vim maven inetutils-ping python-pip && \
   pip install yapf && \
   apt-get -qq clean && \

http://git-wip-us.apache.org/repos/asf/hbase/blob/a1069eb1/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 9cbd8b7..5c03eb9 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -53,7 +53,6 @@ docker build -t hbase_native .
 
 # After the image is built run the thing
 docker run -p 16010:16010/tcp \
-   -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash

[21/50] [abbrv] hbase git commit: HBASE-15401 Add Zookeeper to third party

[eclark]

HBASE-15401 Add Zookeeper to third party


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/27ee8baa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/27ee8baa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/27ee8baa

Branch: refs/heads/HBASE-14850
Commit: 27ee8baaafecfaf4709ecefd1ef2099a82ca3d39
Parents: e00e3a0
Author: Elliott Clark 
Authored: Mon Mar 7 14:58:21 2016 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile   | 26 -
 hbase-native-client/third-party/BUCK | 96 ++-
 2 files changed, 68 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/27ee8baa/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 5f17f04..1364d22 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,8 +15,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman
+FROM cpp_update 
 
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim 
maven inetutils-ping
+ARG CC=/usr/bin/gcc-5
+ARG CXX=/usr/bin/g++-5
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
+
+RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
+RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git 
/usr/src/protobuf && \
+  cd /usr/src/protobuf/ && \
+  ./autogen.sh && \
+  ./configure --disable-shared && \
+  make && \
+  make check && \
+  make install
+RUN cd /usr/src && \
+  wget 
http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz 
&& \ 
+  tar zxf zookeeper-3.4.8.tar.gz && \ 
+  rm -rf zookeeper-3.4.8.tar.gz && \
+  cd zookeeper-3.4.8 && \
+  cd src/c && \
+  ./configure --disable-shared && \
+  make && \
+  make install && \
+  make clean 
 
 WORKDIR /usr/local/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/27ee8baa/hbase-native-client/third-party/BUCK
--
diff --git a/hbase-native-client/third-party/BUCK 
b/hbase-native-client/third-party/BUCK
index b7baa86..e577a5f 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -15,68 +15,60 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-def add_system_libs(names = []):
+def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = 
[], exported_linker_flags = []):
 rules = []
 for name in names:
-prebuilt_cxx_library(
-name = name,
-lib_name = name,
-lib_dir = "/usr/lib/x86_64-linux-gnu",
-visibility = [ 'PUBLIC', ],
-)
-rules.append(":" + name)
-
+gen_rule_name = "gen_lib{}".format(name)
+genrule(
+  name = gen_rule_name,
+  out = gen_rule_name,
+  bash = "mkdir -p $OUT && cp {}/lib{}.* 
$OUT".format(lib_dir,name),
+)
+prebuilt_cxx_library(
+  name = name,
+  lib_name = name,
+  lib_dir = '$(location :{})'.format(gen_rule_name),
+  force_static = True,
+  deps = deps,
+  visibility = [ 'PUBLIC' ],
+  exported_linker_flags = exported_linker_flags,
+)
+rules.append(":" + name)
 return rules
 
 system_libs = [
+"unwind",
+"lzma",
+]
+local_libs = [
 "double-conversion",
 "glog",
-"protobuf",
 "gflags",
-"unwind",
-"lzma",
+"protobuf",
+"zookeeper_mt",
 "boost_regex",
 ]
-tp_dep_rules = add_system_libs(system_libs)
-prebuilt_cxx_library(
-name = "folly",
-lib_name = "folly",
-lib_dir = "/usr/local/lib",
-deps = tp_dep_rules,
-exported_linker_flags = [
-"-pthread",
-"-lstdc++",
-],
-visibility = [
-'PUBLIC',
-]
-)
-prebuilt_cxx_library(
-name = "follybenchmark",
-lib_name = "follybenchmark",
-lib_dir = "/usr/local/lib",
-deps = tp_dep_rules + [":folly"],
-exported_linker_flags = [
-"-pthread",
-"-lstdc++",
-],
-

[36/50] [abbrv] hbase git commit: HBASE-15822 Move to the latest docker base image

[eclark]

HBASE-15822 Move to the latest docker base image


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bb7fa8b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bb7fa8b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bb7fa8b

Branch: refs/heads/HBASE-14850
Commit: 5bb7fa8b832a3134f83b7d94404494ba07f0bd4d
Parents: df14a6a
Author: Elliott Clark 
Authored: Tue May 31 10:42:45 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile   | 2 +-
 hbase-native-client/if/RPC.proto | 1 +
 hbase-native-client/if/WAL.proto | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index be0cbc6..ca87523 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM pjameson/buck-folly-watchman:20160425
+FROM pjameson/buck-folly-watchman:20160511
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5

http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/if/RPC.proto
--
diff --git a/hbase-native-client/if/RPC.proto b/hbase-native-client/if/RPC.proto
index 59bb03d..8413d25 100644
--- a/hbase-native-client/if/RPC.proto
+++ b/hbase-native-client/if/RPC.proto
@@ -125,6 +125,7 @@ message RequestHeader {
   // 0 is NORMAL priority.  200 is HIGH.  If no priority, treat it as NORMAL.
   // See HConstants.
   optional uint32 priority = 6;
+  optional uint32 timeout = 7;
 }
 
 message ResponseHeader {

http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/if/WAL.proto
--
diff --git a/hbase-native-client/if/WAL.proto b/hbase-native-client/if/WAL.proto
index cb9bd8f..2061b22 100644
--- a/hbase-native-client/if/WAL.proto
+++ b/hbase-native-client/if/WAL.proto
@@ -44,9 +44,9 @@ message WALKey {
   required uint64 log_sequence_number = 3;
   required uint64 write_time = 4;
   /*
-  This parameter is deprecated in favor of clusters which 
+  This parameter is deprecated in favor of clusters which
   contains the list of clusters that have consumed the change.
-  It is retained so that the log created by earlier releases (0.94) 
+  It is retained so that the log created by earlier releases (0.94)
   can be read by the newer releases.
   */
   optional UUID cluster_id = 5 [deprecated=true];
@@ -132,6 +132,7 @@ message StoreDescriptor {
   required bytes family_name = 1;
   required string store_home_dir = 2; //relative to region dir
   repeated string store_file = 3; // relative to store dir
+  optional uint64 store_file_size_bytes = 4; // size of store file
 }
 
 /**

[01/50] [abbrv] hbase git commit: HBASE-16171 Fix the potential problems in TestHCM.testConnectionCloseAllowsInterrupt (Colin Ma) [Forced Update!]

[eclark]

Repository: hbase
Updated Branches:
  refs/heads/HBASE-14850 114817253 -> 0eb150f9d (forced update)


HBASE-16171 Fix the potential problems in 
TestHCM.testConnectionCloseAllowsInterrupt (Colin Ma)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e0d97e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e0d97e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e0d97e0

Branch: refs/heads/HBASE-14850
Commit: 5e0d97e0e95434b03ba886279180a031ed997dfd
Parents: a33097e
Author: stack 
Authored: Fri Jul 8 10:17:55 2016 -0700
Committer: stack 
Committed: Fri Jul 8 10:18:36 2016 -0700

--
 .../src/test/java/org/apache/hadoop/hbase/client/TestHCM.java | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5e0d97e0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 4723fa8..1b20b76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -464,6 +464,10 @@ public class TestHCM {
 c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 1); // don't wait between retries.
 c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0); // Server do not really 
expire
 c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
+// to avoid the client to be stuck when do the Get
+c2.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 1);
+c2.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 1);
+c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000);
 
 Connection connection = ConnectionFactory.createConnection(c2);
 final Table table = connection.getTable(tableName);
@@ -488,6 +492,9 @@ public class TestHCM {
 done++;
 if (done % 100 == 0)
   LOG.info("done=" + done);
+// without the sleep, will cause the exception for too many files 
in
+// org.apache.hadoop.hdfs.server.datanode.DataXceiver
+Thread.sleep(100);
   }
 } catch (Throwable t) {
   failed.set(t);

[30/50] [abbrv] hbase git commit: HBASE-15696 Move region location cache serialization into serde

[eclark]

HBASE-15696 Move region location cache serialization into serde

Summary: Use IOBuf for zk

Test Plan: Added a unit test

Differential Revision: https://reviews.facebook.net/D57147


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7cae46e1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7cae46e1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7cae46e1

Branch: refs/heads/HBASE-14850
Commit: 7cae46e117586fbb2336be24b71573c7b5b6ac3a
Parents: dc0f017
Author: Elliott Clark 
Authored: Fri Apr 22 22:21:45 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Dockerfile  |   7 +-
 .../connection/client-dispatcher.h  |   8 ++
 .../connection/client-handler.cc|   7 +-
 hbase-native-client/connection/client-handler.h |   2 +-
 hbase-native-client/connection/pipeline.h   |   2 +-
 hbase-native-client/core/client.cc  |   4 +-
 hbase-native-client/core/client.h   |   4 +-
 hbase-native-client/core/location-cache-test.cc |   2 +-
 hbase-native-client/core/location-cache.cc  |  48 +++-
 hbase-native-client/core/location-cache.h   |   4 +-
 hbase-native-client/core/simple-client.cc   |   6 +-
 hbase-native-client/serde/BUCK  |  10 ++
 .../serde/client-deserializer-test.cc   |   4 +-
 .../serde/client-deserializer.cc|   4 +-
 .../serde/client-serializer-test.cc |   2 +-
 hbase-native-client/serde/client-serializer.cc  |   2 +-
 hbase-native-client/serde/client-serializer.h   |   2 +-
 .../serde/zk-deserializer-test.cc   | 123 +++
 hbase-native-client/serde/zk-deserializer.cc|  78 
 hbase-native-client/serde/zk-deserializer.h |  35 ++
 hbase-native-client/utils/user-util-test.cc |   1 +
 21 files changed, 297 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 9ccc0d3..1524c56 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -19,12 +19,11 @@ FROM pjameson/buck-folly-watchman
 
 ARG CC=/usr/bin/gcc-5
 ARG CXX=/usr/bin/g++-5
-ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
-ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 
-pthread"
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 
-pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 
-pthread"
 
-RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && 
\
+RUN apt-get install -y vim maven inetutils-ping python-pip && \
   pip install yapf && \
-  ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \
   apt-get -qq clean && \
   apt-get -y -qq autoremove && \
   rm -rf /var/lib/{apt,dpkg,cache,log}/ && \

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
index c3987c9..89c7119 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -37,6 +37,14 @@ public:
 
 private:
   std::unordered_map requests_;
+  // Start at some number way above what could
+  // be there for un-initialized call id counters.
+  //
+  // This makes it easier to make sure that the're are
+  // no access to un-initialized variables.
+  //
+  // uint32_t has a max of 4Billion so 10 more or less is
+  // not a big deal.
   uint32_t current_call_id_ = 10;
 };
 } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 74b23ef..7b7794d 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -19,15 +19,15 @@
 
 #include "connection/client-handler.h"
 
-#include 
 #include 
+#include 
 
 #include 
 
 #include "connection/request.h"
 #include "connection/response.h"
-#include "if/RPC.pb.h"
 #include "if/Client.pb.h"
+#include "if/RPC.pb.h"
 
 using namespace hbase;
 using namespace folly;
@@ -60,6 +60,9 @@ void 

[50/50] [abbrv] hbase git commit: HBASE-16174 Hook cell test up, and fix broken cell test.

[eclark]

HBASE-16174 Hook cell test up, and fix broken cell test.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0eb150f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0eb150f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0eb150f9

Branch: refs/heads/HBASE-14850
Commit: 0eb150f9dce5c0dc3dae67a9da7b28f912853952
Parents: fa3ab42
Author: Elliott Clark 
Authored: Tue Jul 5 11:13:01 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/core/BUCK | 8 
 hbase-native-client/core/cell-test.cc | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0eb150f9/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 1c926e3..c615426 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -20,6 +20,7 @@ cxx_library(
 name="core",
 exported_headers=[
 "client.h",
+"cell.h",
 "hbase_macros.h",
 "region-location.h",
 "location-cache.h",
@@ -28,6 +29,7 @@ cxx_library(
 "meta-utils.h",
 ],
 srcs=[
+"cell.cc",
 "client.cc",
 "location-cache.cc",
 "meta-utils.cc",
@@ -51,6 +53,12 @@ cxx_test(name="location-cache-test",
  deps=[":core",
"//test-util:test-util", ],
  run_test_separately=True, )
+cxx_test(name="cell-test",
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
 cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/0eb150f9/hbase-native-client/core/cell-test.cc
--
diff --git a/hbase-native-client/core/cell-test.cc 
b/hbase-native-client/core/cell-test.cc
index cbe50eb..49f567b 100644
--- a/hbase-native-client/core/cell-test.cc
+++ b/hbase-native-client/core/cell-test.cc
@@ -100,7 +100,7 @@ TEST(CellTest, MultipleCellsTest) {
 
 TEST(CellTest, CellRowTest) {
   std::string row = "only-row";
-  std::string family = "";
+  std::string family = "D";
   std::string column = "";
   std::string value = "";
   long timestamp = std::numeric_limits::max();

[32/50] [abbrv] hbase git commit: HBASE-15774 Fix Upgrade lock usage in connection pool.

[eclark]

HBASE-15774 Fix Upgrade lock usage in connection pool.

Summary:
Use upgrade lock better.
Fix a clang warning around initializing the UpgradeHolder incorrectly.
Remove dead code. ( We'l need to add it back when there's a better plan)
Add on more comments.

Test Plan: buck test --all

Differential Revision: https://reviews.facebook.net/D58005


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f6ea4937
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f6ea4937
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f6ea4937

Branch: refs/heads/HBASE-14850
Commit: f6ea49374a552bc321e6954fc3e36587cbf4740f
Parents: eb4cde4
Author: Elliott Clark 
Authored: Tue May 10 17:44:41 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/connection-pool-test.cc  | 12 ++--
 .../connection/connection-pool.cc   | 47 +++---
 .../connection/connection-pool.h|  6 +-
 hbase-native-client/core/client.h   |  2 +-
 hbase-native-client/core/location-cache.cc  | 68 
 5 files changed, 62 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/connection/connection-pool-test.cc
--
diff --git a/hbase-native-client/connection/connection-pool-test.cc 
b/hbase-native-client/connection/connection-pool-test.cc
index c0c346f..bd2d585 100644
--- a/hbase-native-client/connection/connection-pool-test.cc
+++ b/hbase-native-client/connection/connection-pool-test.cc
@@ -79,9 +79,9 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
   sn.set_host_name(hostname);
   sn.set_port(port);
 
-  auto result = cp.get(sn);
+  auto result = cp.Get(sn);
   ASSERT_TRUE(result != nullptr);
-  result = cp.get(sn);
+  result = cp.Get(sn);
 }
 
 TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
@@ -102,13 +102,13 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
   ConnectionPool cp{mock_cf};
 
   {
-auto result_one = cp.get(folly::to(
+auto result_one = cp.Get(folly::to(
 hostname_one + ":" + folly::to(port)));
-auto result_two = cp.get(folly::to(
+auto result_two = cp.Get(folly::to(
 hostname_two + ":" + folly::to(port)));
   }
-  auto result_one = cp.get(
+  auto result_one = cp.Get(
   folly::to(hostname_one + ":" + 
folly::to(port)));
-  auto result_two = cp.get(
+  auto result_two = cp.Get(
   folly::to(hostname_two + ":" + 
folly::to(port)));
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/connection/connection-pool.cc
--
diff --git a/hbase-native-client/connection/connection-pool.cc 
b/hbase-native-client/connection/connection-pool.cc
index 90e2056..aa3d094 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -48,25 +48,54 @@ ConnectionPool::~ConnectionPool() {
   clients_.clear();
 }
 
-std::shared_ptr ConnectionPool::get(const ServerName ) {
-  // Create a read lock.
-  SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+std::shared_ptr ConnectionPool::Get(const ServerName ) {
+  // Try and get th cached connection.
+  auto found_ptr = GetCached(sn);
 
+  // If there's no connection then create it.
+  if (found_ptr == nullptr) {
+found_ptr = GetNew(sn);
+  }
+  return found_ptr;
+}
+
+std::shared_ptr ConnectionPool::GetCached(const ServerName ) {
+  SharedMutexWritePriority::ReadHolder holder(map_mutex_);
   auto found = connections_.find(sn);
-  if (found == connections_.end() || found->second == nullptr) {
-// Move the upgradable lock into the write lock if the connection
-// hasn't been found.
-SharedMutexWritePriority::WriteHolder holder(std::move(holder));
+  if (found == connections_.end()) {
+return nullptr;
+  }
+  return found->second;
+}
+
+std::shared_ptr ConnectionPool::GetNew(const ServerName ) {
+  // Grab the upgrade lock. While we are double checking other readers can
+  // continue on
+  SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
+
+  // Now check if someone else created the connection before we got the lock
+  // This is safe since we hold the upgrade lock.
+  // upgrade lock is more power than the reader lock.
+  auto found = connections_.find(sn);
+  if (found != connections_.end() && found->second != nullptr) {
+return found->second;
+  } else {
+// Yeah it looks a lot like there's no connection
+SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
+
+// Make double sure there are not stale connections hanging around.
+

[38/50] [abbrv] hbase git commit: HBASE-15750 Add on meta deserialization

[eclark]

HBASE-15750 Add on meta deserialization

Summary: Add on meta region info deserialization

Test Plan:
Unit tests.
Simple client connects.

Differential Revision: https://reviews.facebook.net/D57555


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/190fc7aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/190fc7aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/190fc7aa

Branch: refs/heads/HBASE-14850
Commit: 190fc7aae288c97cb23e8cc130dbe7774917cedb
Parents: 50d1672
Author: Elliott Clark 
Authored: Tue May 3 12:17:07 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-dispatcher.cc |   1 +
 .../connection/client-handler.cc|  13 +-
 hbase-native-client/connection/client-handler.h |   6 +-
 .../connection/connection-pool.cc   |  36 +---
 .../connection/connection-pool.h|   8 +-
 hbase-native-client/core/BUCK   |   4 -
 hbase-native-client/core/location-cache-test.cc |   1 +
 hbase-native-client/core/location-cache.cc  |  73 +++-
 hbase-native-client/core/location-cache.h   |   7 +-
 hbase-native-client/core/meta-utils.cc  |   9 +-
 hbase-native-client/core/meta-utils.h   |   7 +-
 hbase-native-client/core/region-location.h  |   5 +-
 hbase-native-client/core/simple-client.cc   |   6 +-
 hbase-native-client/core/table-name-test.cc |  54 --
 hbase-native-client/core/table-name.h   |  50 -
 hbase-native-client/serde/BUCK  |  56 +++---
 .../serde/client-deserializer-test.cc   |  25 ++-
 .../serde/client-deserializer.cc|  68 ---
 hbase-native-client/serde/client-deserializer.h |  36 
 .../serde/client-serializer-test.cc |  26 +--
 hbase-native-client/serde/client-serializer.cc  | 139 --
 hbase-native-client/serde/client-serializer.h   |  55 --
 .../serde/region-info-deserializer-test.cc  |  54 ++
 hbase-native-client/serde/region-info.h |  41 +
 hbase-native-client/serde/rpc.cc| 181 +++
 hbase-native-client/serde/rpc.h |  58 ++
 hbase-native-client/serde/server-name-test.cc   |  32 
 hbase-native-client/serde/server-name.h |  21 +++
 hbase-native-client/serde/table-name-test.cc|  54 ++
 hbase-native-client/serde/table-name.h  |  54 ++
 .../serde/zk-deserializer-test.cc   |   8 +-
 hbase-native-client/serde/zk-deserializer.cc|  78 
 hbase-native-client/serde/zk-deserializer.h |  35 
 hbase-native-client/serde/zk.cc |  78 
 hbase-native-client/serde/zk.h  |  35 
 35 files changed, 761 insertions(+), 653 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/client-dispatcher.cc
--
diff --git a/hbase-native-client/connection/client-dispatcher.cc 
b/hbase-native-client/connection/client-dispatcher.cc
index 817adc1..6e2dc54 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -44,6 +44,7 @@ Future 
ClientDispatcher::operator()(std::unique_ptr arg) {
   auto  = requests_[call_id];
   auto f = p.getFuture();
   p.setInterruptHandler([call_id, this](const folly::exception_wrapper ) {
+LOG(ERROR) << "e = " << call_id;
 this->requests_.erase(call_id);
   });
   this->pipeline_->write(std::move(arg));

http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 3180f4e..496e4f2 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,8 +37,7 @@ using hbase::pb::GetResponse;
 using google::protobuf::Message;
 
 ClientHandler::ClientHandler(std::string user_name)
-: user_name_(user_name), need_send_header_(true), ser_(), deser_(),
-  resp_msgs_() {}
+: user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}
 
 void ClientHandler::read(Context *ctx, std::unique_ptr buf) {
   if (LIKELY(buf != nullptr)) {
@@ -46,7 +45,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr 
buf) {
 Response received;
 ResponseHeader header;
 
-int used_bytes = deser_.parse_delimited(buf.get(), );
+int used_bytes = serde_.ParseDelimited(buf.get(), );
 LOG(INFO) << "Read ResponseHeader size=" << used_bytes
 

[42/50] [abbrv] hbase git commit: HBASE-15926 Added ASF headers which were missed in last commit

[eclark]

HBASE-15926 Added ASF headers which were missed in last commit

Signed-off-by: Elliott Clark 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b6341d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b6341d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b6341d4

Branch: refs/heads/HBASE-14850
Commit: 2b6341d4db3f3cf045b99fb6f71d431098ce7bf1
Parents: 5bb7fa8
Author: Sudeep Sunthankar 
Authored: Thu Jun 2 20:22:37 2016 +1000
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/Makefile| 17 +
 hbase-native-client/Makefile.protos | 17 +
 2 files changed, 34 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2b6341d4/hbase-native-client/Makefile
--
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 7e68b6a..64cef06 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 #use "gcc" to compile source files
 CC:=g++
 LD:=g++

http://git-wip-us.apache.org/repos/asf/hbase/blob/2b6341d4/hbase-native-client/Makefile.protos
--
diff --git a/hbase-native-client/Makefile.protos 
b/hbase-native-client/Makefile.protos
index 4f1727a..2c6316b 100644
--- a/hbase-native-client/Makefile.protos
+++ b/hbase-native-client/Makefile.protos
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 #use "gcc" to compile source files
 CC:=g++
 LD:=g++

[13/50] [abbrv] hbase git commit: HBASE-15604 Add a good readme on the build.

[eclark]

HBASE-15604 Add a good readme on the build.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bfabc715
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bfabc715
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bfabc715

Branch: refs/heads/HBASE-14850
Commit: bfabc715c30350ec1ee1953826ab844ceb60d750
Parents: 45cf565
Author: Elliott Clark 
Authored: Fri Apr 8 13:44:45 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/BUILDING.md | 63 
 1 file changed, 63 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bfabc715/hbase-native-client/BUILDING.md
--
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
new file mode 100644
index 000..20ef2a0
--- /dev/null
+++ b/hbase-native-client/BUILDING.md
@@ -0,0 +1,63 @@
+
+
+#Building HBase native client
+
+The HBase native client build using buck and produces a linux library.
+
+
+# Dependencies
+
+The easiest way to build hbase-native-client is to
+use [Docker](https://www.docker.com/). This will mean that any platform
+with docker can be used to build the hbase-native-client. It will also
+mean that you're building with the same versions of all dependencies that
+the client is tested with.
+
+On OSX the current boot2docker solution will work however it's pretty
+slow. If things get too slow using 
[dinghy](https://github.com/codekitchen/dinghy)
+can help speed things up by using nfs.
+
+If possible pairing virtual box with dinghy will result in the fastest,
+most stable docker environment. However none of it is needed.
+
+# Building using docker
+
+To start with make sure that you have built the java project using
+`mvn package -DskipTests`. That will allow all tests to spin up a standalone
+hbase instance from the jar's created.
+
+Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
+that will build the docker development environment and when complete will
+drop you into a shell on a linux vm with all the tools needed installed.
+
+# Buck
+
+From then on we can use [buck](https://buckbuild.com/) to build everything.
+For example:
+```
+buck build //core:core
+buck test --all
+buck build //core:simple-client
+```
+
+That will build the library, then build and test everything, then build
+the simple-client binary. Buck will find all modules used, and compile 
+them in parallel, caching the results. Output from buck is in the buck-out
+foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs

[14/50] [abbrv] hbase git commit: HBASE-15718 Add on TableName implementation and tests

[eclark]

HBASE-15718 Add on TableName implementation and tests

Summary:
Table name will be needed to look up a row in meta. So this
patch adds on the implementation for creating that lookup
row. It also adds tests for TableName

Test Plan: Added on a unit test

Differential Revision: https://reviews.facebook.net/D57285


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2aaadab1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2aaadab1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2aaadab1

Branch: refs/heads/HBASE-14850
Commit: 2aaadab1519cdb11dd2c8df6cc8e61c1ecf4e82d
Parents: 7cae46e
Author: Elliott Clark 
Authored: Tue Apr 26 14:51:06 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-handler.cc|  1 -
 hbase-native-client/core/BUCK   | 69 +++-
 hbase-native-client/core/meta-utils.cc  | 31 +
 hbase-native-client/core/meta-utils.h   | 30 +
 hbase-native-client/core/simple-client.cc   |  4 +-
 hbase-native-client/core/table-name-test.cc | 50 ++
 hbase-native-client/core/table-name.cc  | 15 +
 hbase-native-client/core/table-name.h   | 25 ++-
 hbase-native-client/if/Master.proto | 13 
 hbase-native-client/if/ZooKeeper.proto  | 10 ++-
 10 files changed, 211 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/connection/client-handler.cc
--
diff --git a/hbase-native-client/connection/client-handler.cc 
b/hbase-native-client/connection/client-handler.cc
index 7b7794d..205993a7 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -60,7 +60,6 @@ void ClientHandler::read(Context *ctx, std::unique_ptr 
buf) {
   }
 }
 
-
 // TODO(eclark): Figure out how to handle the
 // network errors that are going to come.
 Future ClientHandler::write(Context *ctx, Request r) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 195fc5c..9db6fda 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -16,44 +16,49 @@
 # limitations under the License.
 
 # This is the main library.
-cxx_library(name="core",
-exported_headers=[
-"client.h",
-"connection.h",
-"get-request.h",
-"get-result.h",
-"hbase_macros.h",
-"location-cache.h",
-"table-name.h",
-],
-srcs=[
-"client.cc",
-"get-request.cc",
-"get-result.cc",
-"location-cache.cc",
-"table-name.cc",
-],
-deps=[
-"//connection:connection",
-"//if:if",
-"//serde:serde",
-"//third-party:folly",
-"//third-party:wangle",
-"//third-party:zookeeper_mt",
-],
-visibility=[
-'PUBLIC',
-], )
-
+cxx_library(
+name="core",
+exported_headers=[
+"client.h",
+"connection.h",
+"get-request.h",
+"get-result.h",
+"hbase_macros.h",
+"location-cache.h",
+"table-name.h",
+# TODO: move this out of exported
+# Once meta lookup works
+"meta-utils.h",
+],
+srcs=[
+"client.cc",
+"get-request.cc",
+"get-result.cc",
+"location-cache.cc",
+"meta-utils.cc",
+"table-name.cc",
+],
+deps=[
+"//connection:connection",
+"//if:if",
+"//serde:serde",
+"//third-party:folly",
+"//third-party:wangle",
+"//third-party:zookeeper_mt",
+],
+visibility=[
+'PUBLIC',
+], )
 cxx_test(name="location-cache-test",
  srcs=[
  "test-env.cc",
  "location-cache-test.cc",
  ],
- deps=[
- ":core",
- ],
+ deps=[":core", ],
  run_test_separately=True, )
+cxx_test(name="table-name-test",
+ srcs=["table-name-test.cc", ],
+ deps=[":core", ], )
 cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/meta-utils.cc

[16/50] [abbrv] hbase git commit: HBASE-14852 Update build env

[eclark]

http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_result.h
--
diff --git a/hbase-native-client/src/async/hbase_result.h 
b/hbase-native-client/src/async/hbase_result.h
deleted file mode 100644
index eecbbb3..000
--- a/hbase-native-client/src/async/hbase_result.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_RESULT_H_
-#define ASYNC_HBASE_RESULT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-HBASE_API int32_t hb_result_destroy(hb_result_t result);
-
-HBASE_API int32_t hb_result_get_cells(hb_result_t result,
-hb_cell_t ** cell_ptr, size_t * num_cells);
-
-HBASE_API int32_t hb_result_get_table(hb_result_t result,
-char ** table, size_t * table_length);
-HBASE_API int32_t hb_result_get_namespace(hb_result_t result,
-char ** name_space, size_t * name_space_length);
-
-#ifdef __cplusplus
-}  // extern "C"
-#endif  // __cplusplus
-
-#endif  // ASYNC_HBASE_RESULT_H_

http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_scanner.cc
--
diff --git a/hbase-native-client/src/async/hbase_scanner.cc 
b/hbase-native-client/src/async/hbase_scanner.cc
deleted file mode 100644
index 5a8e555..000
--- a/hbase-native-client/src/async/hbase_scanner.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "async/hbase_scanner.h"
-
-#include 
-
-#include "core/hbase_types.h"
-#include "core/scanner.h"
-
-int32_t hb_scanner_create(hb_scanner_t * scanner_ptr) {
-  (*scanner_ptr) = reinterpret_cast(new Scanner());
-  return (*scanner_ptr != NULL)?0:1;
-}
-
-HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
-char * table, size_t table_length) {
-  return 0;
-}
-
-HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
-char * name_space, size_t name_space_length) {
-  return 0;
-}
-
-int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
-unsigned char * start_row, size_t start_row_length) {
-  return 0;
-}
-
-int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
-unsigned char * end_row, size_t end_row_length) {
-  return 0;
-}
-
-int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
-size_t cache_size) {
-  return 0;
-}
-
-int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
-int8_t num_versions) {
-  return 0;
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_scanner.h
--
diff --git a/hbase-native-client/src/async/hbase_scanner.h 
b/hbase-native-client/src/async/hbase_scanner.h
deleted file mode 100644
index cd3f544..000
--- a/hbase-native-client/src/async/hbase_scanner.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * 

[06/50] [abbrv] hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells

[eclark]

HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9cf012cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9cf012cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9cf012cd

Branch: refs/heads/HBASE-14850
Commit: 9cf012cd0086e054a66f54cb54771af2381fade0
Parents: 6329697
Author: Yu Li 
Authored: Sun Jul 10 13:45:17 2016 +0800
Committer: Yu Li 
Committed: Sun Jul 10 13:45:17 2016 +0800

--
 .../hbase/regionserver/AbstractMemStore.java| 15 +++
 .../hbase/regionserver/HeapMemStoreLAB.java | 11 +
 .../hbase/regionserver/MemStoreCompactor.java   |  3 ++-
 .../hbase/regionserver/MutableSegment.java  |  6 +++--
 .../hadoop/hbase/regionserver/Segment.java  | 23 ++---
 .../hbase/regionserver/TestDefaultMemStore.java | 26 
 6 files changed, 73 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index d25f960..79e95af 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,7 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-return internalAdd(toAdd);
+boolean useMSLAB = (toAdd != cell);
+return internalAdd(toAdd, useMSLAB);
   }
 
   /**
@@ -156,7 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long delete(Cell deleteCell) {
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-long s = internalAdd(toAdd);
+boolean useMSLAB = (toAdd != deleteCell);
+long s = internalAdd(toAdd, useMSLAB);
 return s;
   }
 
@@ -243,7 +245,7 @@ public abstract class AbstractMemStore implements MemStore {
 // hitting OOME - see TestMemStore.testUpsertMSLAB for a
 // test that triggers the pathological case if we don't avoid MSLAB
 // here.
-long addedSize = internalAdd(cell);
+long addedSize = internalAdd(cell, false);
 
 // Get the Cells for the row/family/qualifier regardless of timestamp.
 // For this case we want to clean up any other puts
@@ -387,9 +389,12 @@ public abstract class AbstractMemStore implements MemStore 
{
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+   * @param toAdd the cell to add
+   * @param useMSLAB whether using MSLAB
+   * @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd) {
-long s = active.add(toAdd);
+  private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
+long s = active.add(toAdd, useMSLAB);
 setOldestEditTimeToNow();
 checkActiveSize();
 return s;

http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
index f22a6e5..625811a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.util.ByteRange;
 import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
 /**
@@ -206,6 +207,11 @@ public class HeapMemStoreLAB implements MemStoreLAB {
 }
   }
 
+  @VisibleForTesting
+  Chunk getCurrentChunk() {
+return this.curChunk.get();
+  }
+
   /**
* A chunk of memory out of which allocations are sliced.
*/
@@ -311,5 +317,10 @@ public class HeapMemStoreLAB implements MemStoreLAB {
 " allocs=" + allocCount.get() + "waste=" +
 (data.length - nextFreeOffset.get());
 }
+
+@VisibleForTesting
+int getNextFreeOffset() {
+  return this.nextFreeOffset.get();
+}
   }
 }

[11/50] [abbrv] hbase git commit: HBASE-15823 Use call once for user util

[eclark]

HBASE-15823 Use call once for user util


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dea1eb64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dea1eb64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dea1eb64

Branch: refs/heads/HBASE-14850
Commit: dea1eb64bd6ef85ef0ca9a4c1f5c7b30043ba953
Parents: e718c7d
Author: Elliott Clark 
Authored: Fri May 13 13:07:03 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/utils/user-util.cc | 13 ++---
 hbase-native-client/utils/user-util.h  |  6 ++
 2 files changed, 4 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/dea1eb64/hbase-native-client/utils/user-util.cc
--
diff --git a/hbase-native-client/utils/user-util.cc 
b/hbase-native-client/utils/user-util.cc
index 3d963b3..9e170e0 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -27,22 +27,14 @@
 using namespace hbase;
 using namespace std;
 
-UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
+UserUtil::UserUtil() : once_flag_{}, user_name_{"drwho"} {}
 
 string UserUtil::user_name() {
-  if (!init_) {
-compute_user_name();
-  }
+  std::call_once(once_flag_, [this]() { compute_user_name(); });
   return user_name_;
 }
 
 void UserUtil::compute_user_name() {
-  lock_guard lock(m_);
-
-  if (init_) {
-return;
-  }
-
   // According to the man page of getpwuid
   // this should never be free'd
   //
@@ -52,6 +44,5 @@ void UserUtil::compute_user_name() {
   // make sure that we got something.
   if (passwd && passwd->pw_name) {
 user_name_ = string{passwd->pw_name};
-init_ = true;
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/dea1eb64/hbase-native-client/utils/user-util.h
--
diff --git a/hbase-native-client/utils/user-util.h 
b/hbase-native-client/utils/user-util.h
index 0b4cc73..fdfc0c8 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -19,8 +19,7 @@
 
 #pragma once
 
-#include 
-#include 
+#include 
 #include 
 
 namespace hbase {
@@ -49,8 +48,7 @@ private:
* Compute the username. This will block.
*/
   void compute_user_name();
-  std::atomic init_;
+  std::once_flag once_flag_;
   std::string user_name_;
-  std::mutex m_;
 };
 } // namespace hbase

[25/50] [abbrv] hbase git commit: HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.

[eclark]

HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.

Summary: On every start of docker make sure that protos are up to date.

Test Plan: Added the script and it updated the protos, but didn't delete BUCK.

Differential Revision: https://reviews.facebook.net/D56763


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bbeaacb6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bbeaacb6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bbeaacb6

Branch: refs/heads/HBASE-14850
Commit: bbeaacb6d43b29746abc78f31c422c7706938f8a
Parents: 098028c
Author: Elliott Clark 
Authored: Wed Apr 13 15:59:13 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/copy-protobuf.sh |   8 ++
 hbase-native-client/bin/start-docker.sh  |  22 +++-
 hbase-native-client/if/Admin.proto   |   1 +
 hbase-native-client/if/Client.proto  |   8 +-
 hbase-native-client/if/ClusterStatus.proto   |   3 +
 hbase-native-client/if/Filter.proto  |   1 +
 hbase-native-client/if/Master.proto  |  47 +++-
 hbase-native-client/if/MasterProcedure.proto |  40 +++
 hbase-native-client/if/RSGroup.proto |  34 ++
 hbase-native-client/if/RSGroupAdmin.proto| 136 ++
 hbase-native-client/if/ZooKeeper.proto   |  13 +++
 11 files changed, 304 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/bin/copy-protobuf.sh
--
diff --git a/hbase-native-client/bin/copy-protobuf.sh 
b/hbase-native-client/bin/copy-protobuf.sh
new file mode 100755
index 000..c9e70f2
--- /dev/null
+++ b/hbase-native-client/bin/copy-protobuf.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+BIN_DIR=$(dirname "$0")
+PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
+PB_DEST_DIR="${BIN_DIR}/../if/"
+rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}

http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 725ed6a..9cbd8b7 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -23,22 +23,38 @@ set -x
 eval "$(docker-machine env docker-vm)"
 eval "$(docker-machine env dinghy)"
 
-# Build the image
-docker build -t hbase_native .
+BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd  2>&1 > 
/dev/null)
+BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd  2>&1 > 
/dev/null)
 
+${BIN_DIR}/copy-protobuf.sh
 
+# Go into the base dir. This just makes things cleaner.
+pushd ${BASE_DIR}
+
+# Make sure that there is a thrid-party dir.
 mkdir third-party || true
+
+# Get gtest
+# TODO(eclark): Remove this ( see HBASE-15427 )
 if [[ ! -d third-party/googletest ]]; then
 git clone https://github.com/google/googletest.git 
third-party/googletest
 fi
 
+# We don't want to have to re-download all the jars in docker if we can help it
 if [[ ! -d ~/.m2 ]]; then
 echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
 exit 1
 fi;
 
+# Build the image
+# 
+# This shouldn't be needed after the development environment is a little more 
stable.
+docker build -t hbase_native .
+
+# After the image is built run the thing
 docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-   -v ${PWD}/..:/usr/src/hbase \
+   -v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native  /bin/bash
+popd

http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Admin.proto
--
diff --git a/hbase-native-client/if/Admin.proto 
b/hbase-native-client/if/Admin.proto
index e905340..a1905a4 100644
--- a/hbase-native-client/if/Admin.proto
+++ b/hbase-native-client/if/Admin.proto
@@ -25,6 +25,7 @@ option java_generic_services = true;
 option java_generate_equals_and_hash = true;
 option optimize_for = SPEED;
 
+import "Client.proto";
 import "HBase.proto";
 import "WAL.proto";
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Client.proto
--
diff --git a/hbase-native-client/if/Client.proto 
b/hbase-native-client/if/Client.proto
index ca9abdc..8a4d459 100644
--- a/hbase-native-client/if/Client.proto
+++ b/hbase-native-client/if/Client.proto
@@ -402,6 +402,11 @@ message 

[31/50] [abbrv] hbase git commit: HBASE-15821 Document TestUtil

[eclark]

HBASE-15821 Document TestUtil


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8014cdfe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8014cdfe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8014cdfe

Branch: refs/heads/HBASE-14850
Commit: 8014cdfe9d716eae92a91a7f376c002c05446ff9
Parents: dea1eb6
Author: Elliott Clark 
Authored: Fri May 13 09:17:49 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/bin/format-code.sh  |  2 +-
 .../connection/connection-pool.h|  4 ++--
 hbase-native-client/core/location-cache.cc  |  2 +-
 hbase-native-client/core/location-cache.h   |  2 +-
 hbase-native-client/test-util/test-util.cc  | 20 +---
 hbase-native-client/test-util/test-util.h   | 20 +++-
 6 files changed, 33 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/bin/format-code.sh
--
diff --git a/hbase-native-client/bin/format-code.sh 
b/hbase-native-client/bin/format-code.sh
index 36a114d..cc8b368 100755
--- a/hbase-native-client/bin/format-code.sh
+++ b/hbase-native-client/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
 IFS=$'\n\t'
 
 
-find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 
clang-format -i
+find core connection serde utils test-util -name "*.h" -or -name "*.cc" | 
xargs -P8 clang-format -i
 find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/connection/connection-pool.h
--
diff --git a/hbase-native-client/connection/connection-pool.h 
b/hbase-native-client/connection/connection-pool.h
index 605a81b..b8c950b 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -88,8 +88,8 @@ public:
   void Close(const hbase::pb::ServerName );
 
 private:
-  std::shared_ptr GetCached(const hbase::pb::ServerName& sn);
-  std::shared_ptr GetNew(const hbase::pb::ServerName& sn);
+  std::shared_ptr GetCached(const hbase::pb::ServerName );
+  std::shared_ptr GetNew(const hbase::pb::ServerName );
   std::unordered_map
   connections_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/core/location-cache.cc
--
diff --git a/hbase-native-client/core/location-cache.cc 
b/hbase-native-client/core/location-cache.cc
index 6ba8add..efd2210 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName , const 
string ) {
   return this->LocateMeta()
   .via(cpu_executor_.get())
   .then([this](ServerName sn) { return this->cp_.Get(sn); })
-.then([tn, row, this](std::shared_ptr service) {
+  .then([tn, row, this](std::shared_ptr service) {
 return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
   })
   .then([this](Response resp) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/core/location-cache.h
--
diff --git a/hbase-native-client/core/location-cache.h 
b/hbase-native-client/core/location-cache.h
index d435530..830cd96 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -55,7 +55,7 @@ public:
* @param io_executor executor used to talk to the network
*/
   LocationCache(std::string quorum_spec,
-std::shared_ptr cpu_exector,
+std::shared_ptr cpu_executor,
 std::shared_ptr io_executor);
   /**
* Destructor.

http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/test-util/test-util.cc
--
diff --git a/hbase-native-client/test-util/test-util.cc 
b/hbase-native-client/test-util/test-util.cc
index e5fba48..88ce7c8 100644
--- a/hbase-native-client/test-util/test-util.cc
+++ b/hbase-native-client/test-util/test-util.cc
@@ -24,13 +24,18 @@
 using hbase::TestUtil;
 using folly::Random;
 
-const static int STR_LEN = 32;
-
-std::string TestUtil::RandString() {
-  auto s = std::string(STR_LEN, 'z');
-
-  for (int i = 0; i < STR_LEN; i++) {
+std::string TestUtil::RandString(int len) {
+  // Create the whole string.
+  // 

[05/50] [abbrv] hbase git commit: HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

[eclark]

HBASE-14548 Expand how table coprocessor jar and dependency path can be 
specified (Xiang Li)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63296978
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63296978
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63296978

Branch: refs/heads/HBASE-14850
Commit: 632969787aa9e9e1d73b83449b93e21f91110b73
Parents: 496fd98
Author: Jerry He 
Authored: Sat Jul 9 18:00:41 2016 -0700
Committer: Jerry He 
Committed: Sat Jul 9 18:01:49 2016 -0700

--
 .../hbase/util/CoprocessorClassLoader.java  | 72 +---
 .../hbase/util/TestCoprocessorClassLoader.java  | 43 
 src/main/asciidoc/_chapters/cp.adoc |  9 ++-
 3 files changed, 97 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 11016c3..c3635cb 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.io.IOUtils;
 
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase 
{
 super(parent);
   }
 
-  private void init(Path path, String pathPrefix,
+  private void init(Path pathPattern, String pathPrefix,
   Configuration conf) throws IOException {
 // Copy the jar to the local filesystem
 String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends 
ClassLoaderBase {
   }
 }
 
-FileSystem fs = path.getFileSystem(conf);
-File dst = new File(parentDirStr, "." + pathPrefix + "."
-  + path.getName() + "." + System.currentTimeMillis() + ".jar");
-fs.copyToLocalFile(path, new Path(dst.toString()));
-dst.deleteOnExit();
-
-addURL(dst.getCanonicalFile().toURI().toURL());
-
-JarFile jarFile = new JarFile(dst.toString());
-try {
-  Enumeration entries = jarFile.entries();
-  while (entries.hasMoreElements()) {
-JarEntry entry = entries.nextElement();
-Matcher m = libJarPattern.matcher(entry.getName());
-if (m.matches()) {
-  File file = new File(parentDirStr, "." + pathPrefix + "."
-+ path.getName() + "." + System.currentTimeMillis() + "." + 
m.group(1));
-  try (FileOutputStream outStream = new FileOutputStream(file)) {
-IOUtils.copyBytes(jarFile.getInputStream(entry),
-  outStream, conf, true);
+FileSystem fs = pathPattern.getFileSystem(conf);
+Path pathPattern1 = fs.isDirectory(pathPattern) ?
+  new Path(pathPattern, "*.jar") : pathPattern;  // append "*.jar" if a 
directory is specified
+FileStatus[] fileStatuses = fs.globStatus(pathPattern1);  // return all 
files that match the pattern
+if (fileStatuses == null || fileStatuses.length == 0) {  // if no one 
matches
+  throw new FileNotFoundException(pathPattern1.toString());
+} else {
+  boolean validFileEncountered = false;
+  for (Path path : FileUtil.stat2Paths(fileStatuses)) {  // for each file 
that match the pattern
+if (fs.isFile(path)) {  // only process files, skip for directories
+  File dst = new File(parentDirStr, "." + pathPrefix + "."
++ path.getName() + "." + System.currentTimeMillis() + ".jar");
+  fs.copyToLocalFile(path, new Path(dst.toString()));
+  dst.deleteOnExit();
+
+  addURL(dst.getCanonicalFile().toURI().toURL());
+
+  JarFile jarFile = new JarFile(dst.toString());
+  try {
+Enumeration entries = jarFile.entries();  // get entries 
inside a jar file
+while (entries.hasMoreElements()) {
+  JarEntry entry = entries.nextElement();
+  Matcher m = libJarPattern.matcher(entry.getName());
+  if (m.matches()) {
+File file = new 

[17/50] [abbrv] hbase git commit: HBASE-14852 Update build env

[eclark]

HBASE-14852 Update build env

Also includes HBASE-14858 Clean up so core is ready for development on a recent 
version of c++


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8556e3a3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8556e3a3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8556e3a3

Branch: refs/heads/HBASE-14850
Commit: 8556e3a348fb4e4d1610ab44084b70631331a98a
Parents: a396ae7
Author: Elliott Clark 
Authored: Thu Nov 19 16:28:11 2015 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/.buckconfig |   2 +
 hbase-native-client/.gitignore  |  23 +--
 hbase-native-client/CMakeLists.txt  | 157 ---
 hbase-native-client/Dockerfile  |  22 +++
 hbase-native-client/README.md   |  15 +-
 hbase-native-client/bin/build-all.sh|  41 -
 hbase-native-client/bin/build-thirdparty.sh |  64 
 hbase-native-client/bin/download-thirdparty.sh  |  70 -
 hbase-native-client/bin/hbase-client-env.sh |  47 --
 hbase-native-client/bin/start-docker.sh |  32 
 .../cmake_modules/FindGTest.cmake   |  53 ---
 .../cmake_modules/FindLibEv.cmake   |  47 --
 hbase-native-client/core/BUCK   |  50 ++
 hbase-native-client/core/admin.cc   |  20 +++
 hbase-native-client/core/admin.h|  22 +++
 hbase-native-client/core/client.cc  |  38 +
 hbase-native-client/core/client.h   |  24 +++
 hbase-native-client/core/connection.cc  |  20 +++
 hbase-native-client/core/connection.h   |  26 +++
 hbase-native-client/core/connection_attr.h  |  24 +++
 hbase-native-client/core/delete.cc  |  21 +++
 hbase-native-client/core/delete.h   |  27 
 hbase-native-client/core/get.cc |  20 +++
 hbase-native-client/core/get.h  |  22 +++
 hbase-native-client/core/hbase_macros.h |  56 +++
 hbase-native-client/core/mutation.cc|  41 +
 hbase-native-client/core/mutation.h |  58 +++
 hbase-native-client/core/put.cc |  21 +++
 hbase-native-client/core/put.h  |  27 
 hbase-native-client/core/scanner.cc |  20 +++
 hbase-native-client/core/scanner.h  |  22 +++
 hbase-native-client/rpc/CMakeLists.txt  |  17 ++
 hbase-native-client/src/async/CMakeLists.txt|  32 
 hbase-native-client/src/async/get-test.cc   |  59 ---
 hbase-native-client/src/async/hbase_admin.cc|  57 ---
 hbase-native-client/src/async/hbase_admin.h |  69 
 hbase-native-client/src/async/hbase_client.cc   |  47 --
 hbase-native-client/src/async/hbase_client.h|  60 ---
 .../src/async/hbase_connection.cc   |  37 -
 .../src/async/hbase_connection.h|  52 --
 hbase-native-client/src/async/hbase_errno.h |  23 ---
 hbase-native-client/src/async/hbase_get.cc  |  61 ---
 hbase-native-client/src/async/hbase_get.h   |  73 -
 .../src/async/hbase_mutations.cc| 111 -
 hbase-native-client/src/async/hbase_mutations.h | 119 --
 hbase-native-client/src/async/hbase_result.cc   |  37 -
 hbase-native-client/src/async/hbase_result.h|  44 --
 hbase-native-client/src/async/hbase_scanner.cc  |  59 ---
 hbase-native-client/src/async/hbase_scanner.h   |  80 --
 hbase-native-client/src/async/mutations-test.cc | 102 
 hbase-native-client/src/core/CMakeLists.txt |  31 
 hbase-native-client/src/core/admin.cc   |  20 ---
 hbase-native-client/src/core/admin.h|  25 ---
 hbase-native-client/src/core/client.cc  |  20 ---
 hbase-native-client/src/core/client.h   |  25 ---
 hbase-native-client/src/core/connection.cc  |  22 ---
 hbase-native-client/src/core/connection.h   |  26 ---
 hbase-native-client/src/core/connection_attr.h  |  30 
 hbase-native-client/src/core/delete.cc  |  22 ---
 hbase-native-client/src/core/delete.h   |  29 
 hbase-native-client/src/core/get.cc |  20 ---
 hbase-native-client/src/core/get.h  |  26 ---
 .../src/core/hbase_connection_attr.cc   |  41 -
 .../src/core/hbase_connection_attr.h|  51 --
 hbase-native-client/src/core/hbase_macros.h |  60 ---
 hbase-native-client/src/core/hbase_types.h  |  83 --
 hbase-native-client/src/core/mutation.cc|  42 -
 hbase-native-client/src/core/mutation.h |  48 --
 hbase-native-client/src/core/put.cc |  22 ---
 hbase-native-client/src/core/put.h

[28/50] [abbrv] hbase git commit: HBASE-15620 Add on Call serialization

[eclark]

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/BUCK
--
diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
new file mode 100644
index 000..207607f
--- /dev/null
+++ b/hbase-native-client/serde/BUCK
@@ -0,0 +1,54 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="serde",
+exported_headers=[
+"client-serializer.h",
+"client-deserializer.h",
+],
+srcs=[
+"client-serializer.cc",
+"client-deserializer.cc",
+],
+deps=[
+"//if:if",
+"//third-party:folly",
+],
+tests=[
+":client-serializer-test",
+":client-deserializer-test",
+],
+visibility=[
+'PUBLIC',
+], )
+
+cxx_test(name="client-serializer-test",
+ srcs=[
+ "client-serializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
+cxx_test(name="client-deserializer-test",
+ srcs=[
+ "client-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-deserializer-test.cc
--
diff --git a/hbase-native-client/serde/client-deserializer-test.cc 
b/hbase-native-client/serde/client-deserializer-test.cc
new file mode 100644
index 000..bb57e50
--- /dev/null
+++ b/hbase-native-client/serde/client-deserializer-test.cc
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include 
+#include 
+
+#include "serde/client-deserializer.h"
+#include "serde/client-serializer.h"
+#include "if/Client.pb.h"
+
+using namespace hbase;
+using folly::IOBuf;
+using hbase::pb::GetRequest;
+using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+TEST(TestClientDeserializer, TestReturnFalseOnNullPtr) {
+  ClientDeserializer deser;
+  ASSERT_LT(deser.parse_delimited(nullptr, nullptr), 0);
+}
+
+TEST(TestClientDeserializer, TestReturnFalseOnBadInput) {
+  ClientDeserializer deser;
+  auto buf = IOBuf::copyBuffer("test");
+  GetRequest gr;
+
+  ASSERT_LT(deser.parse_delimited(buf.get(), ), 0);
+}
+
+TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
+  GetRequest in;
+  ClientSerializer ser;
+  ClientDeserializer deser;
+
+  // fill up the GetRequest.
+  in.mutable_region()->set_value("test_region_id");
+  in.mutable_region()->set_type(
+  RegionSpecifier_RegionSpecifierType::
+  RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+  in.mutable_get()->set_row("test_row");
+
+  // Create the buffer
+  auto buf = ser.serialize_delimited(in);
+
+  GetRequest out;
+
+  int used_bytes = deser.parse_delimited(buf.get(), );
+
+  ASSERT_GT(used_bytes, 0);
+  ASSERT_EQ(used_bytes, buf->length());
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-deserializer.cc
--
diff --git a/hbase-native-client/serde/client-deserializer.cc 
b/hbase-native-client/serde/client-deserializer.cc
new file 

[24/50] [abbrv] hbase git commit: HBASE-15418 Clean up un-used warning in test util

[eclark]

HBASE-15418 Clean up un-used warning in test util


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/45cf5657
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/45cf5657
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/45cf5657

Branch: refs/heads/HBASE-14850
Commit: 45cf5657acc4cc7a61655fa6b76eb90910fabb21
Parents: 26ba4ed
Author: Elliott Clark 
Authored: Fri Mar 25 15:44:06 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../bin/start_local_hbase_and_wait.sh   |  9 +-
 .../bin/stop_local_hbase_and_wait.sh|  2 +-
 hbase-native-client/core/BUCK   |  6 
 .../core/native-client-test-env.cc  |  9 --
 hbase-native-client/core/test_env.h | 32 
 5 files changed, 15 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/bin/start_local_hbase_and_wait.sh
--
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh 
b/hbase-native-client/bin/start_local_hbase_and_wait.sh
index 64d0b68..cfc71f9 100755
--- a/hbase-native-client/bin/start_local_hbase_and_wait.sh
+++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
@@ -17,10 +17,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Clean up from any other tests.
+rm -rf /tmp/hbase-*
+
+# Start the master/regionservers.
 $PWD/../bin/start-hbase.sh
 
-until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == 
"200" ]
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" 
http://localhost:16010/jmx) == "200" ]
 do
  printf "Waiting for local HBase cluster to start\n"
  sleep 1
 done
+
+# This sucks, but master can easily be up and meta not be assigned yet.
+sleep 30

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/bin/stop_local_hbase_and_wait.sh
--
diff --git a/hbase-native-client/bin/stop_local_hbase_and_wait.sh 
b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
index 4e89334..761412a 100755
--- a/hbase-native-client/bin/stop_local_hbase_and_wait.sh
+++ b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
@@ -17,7 +17,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-$PWD/../bin/stop-hbase.sh
+ps aux | grep proc_master | awk '{print $2}' | xargs kill -9
 
 while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == 
"200" ]
 do

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/BUCK
--
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 817b5a0..d1e89d1 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -51,9 +51,6 @@ cxx_library(name="core",
 ], )
 
 cxx_test(name="simple-test",
- headers=[
- "test_env.h",
- ],
  srcs=[
  "native-client-test-env.cc",
  "simple-native-client-test.cc",
@@ -63,9 +60,6 @@ cxx_test(name="simple-test",
  ],
  run_test_separately=True, )
 cxx_test(name="location-cache-test",
- headers=[
- "test_env.h",
- ],
  srcs=[
  "native-client-test-env.cc",
  "location-cache-test.cc",

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/native-client-test-env.cc
--
diff --git a/hbase-native-client/core/native-client-test-env.cc 
b/hbase-native-client/core/native-client-test-env.cc
index a86961f..07f30a6 100644
--- a/hbase-native-client/core/native-client-test-env.cc
+++ b/hbase-native-client/core/native-client-test-env.cc
@@ -18,18 +18,21 @@
  */
 
 #include 
-#include 
 
 namespace {
 
 class NativeClientTestEnv : public ::testing::Environment {
  public:
   void SetUp() override {
-init_test_env();
+// start local HBase cluster to be reused by all tests
+auto result = system("bin/start_local_hbase_and_wait.sh");
+ASSERT_EQ(0, result);
   }
 
   void TearDown() override {
-clean_test_env();
+// shutdown local HBase cluster
+auto result = system("bin/stop_local_hbase_and_wait.sh");
+ASSERT_EQ(0, result);
   }
 };
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/test_env.h
--
diff --git a/hbase-native-client/core/test_env.h 

[15/50] [abbrv] hbase git commit: HBASE-15739 Add region location lookup from meta

[eclark]

HBASE-15739 Add region location lookup from meta

Summary:
Add the ability for location-cache to read from meta.
There's still no cache. Or any parsing.

Test Plan: Needs tests.

Differential Revision: https://reviews.facebook.net/D57489


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f74379e6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f74379e6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f74379e6

Branch: refs/heads/HBASE-14850
Commit: f74379e6875a3f64c185eaefd0cb4594f9ca8022
Parents: baf863f
Author: Elliott Clark 
Authored: Sat Apr 30 17:12:11 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/connection/BUCK |  1 +
 .../connection/client-dispatcher.cc |  2 +
 .../connection/client-dispatcher.h  |  3 +-
 .../connection/client-handler.cc|  5 +-
 hbase-native-client/connection/client-handler.h |  2 +-
 .../connection/connection-factory.cc|  2 +-
 .../connection/connection-factory.h |  1 +
 .../connection/connection-pool.cc   | 14 ++---
 hbase-native-client/connection/pipeline.cc  |  2 +
 hbase-native-client/connection/pipeline.h   |  1 +
 hbase-native-client/connection/response.h   |  6 ++-
 hbase-native-client/core/BUCK   |  3 +-
 hbase-native-client/core/location-cache.cc  | 37 --
 hbase-native-client/core/location-cache.h   | 19 ++-
 hbase-native-client/core/meta-utils.cc  | 54 +++-
 hbase-native-client/core/meta-utils.h   |  9 +++-
 hbase-native-client/core/region-location.h  | 45 
 hbase-native-client/core/simple-client.cc   | 48 ++---
 hbase-native-client/core/table-name-test.cc | 28 +-
 hbase-native-client/core/table-name.cc  | 34 
 hbase-native-client/core/table-name.h   | 35 ++---
 hbase-native-client/serde/BUCK  |  1 +
 hbase-native-client/utils/BUCK  |  7 +--
 hbase-native-client/utils/user-util.cc  |  2 +
 hbase-native-client/utils/user-util.h   |  3 +-
 25 files changed, 231 insertions(+), 133 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/BUCK
--
diff --git a/hbase-native-client/connection/BUCK 
b/hbase-native-client/connection/BUCK
index 96f2136..39b8be5 100644
--- a/hbase-native-client/connection/BUCK
+++ b/hbase-native-client/connection/BUCK
@@ -43,6 +43,7 @@ cxx_library(name="connection",
 "//third-party:folly",
 "//third-party:wangle",
 ],
+compiler_flags=['-Weffc++'],
 visibility=['//core/...', ], )
 cxx_test(name="connection-pool-test",
  srcs=["connection-pool-test.cc", ],

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/client-dispatcher.cc
--
diff --git a/hbase-native-client/connection/client-dispatcher.cc 
b/hbase-native-client/connection/client-dispatcher.cc
index eea0a17..817adc1 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -22,6 +22,8 @@ using namespace folly;
 using namespace hbase;
 using namespace wangle;
 
+ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
+
 void ClientDispatcher::read(Context *ctx, Response in) {
   auto call_id = in.call_id();
   auto search = requests_.find(call_id);

http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
index 877e877..826fc6a 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -30,6 +30,7 @@ class ClientDispatcher
 : public wangle::ClientDispatcherBase {
 public:
+  ClientDispatcher();
   void read(Context *ctx, Response in) override;
   folly::Future operator()(std::unique_ptr arg) override;
   folly::Future close(Context *ctx) override;
@@ -45,6 +46,6 @@ private:
   //
   // uint32_t has a max of 4Billion so 10 more or less is
   // not a big deal.
-  uint32_t current_call_id_ = 10;
+  uint32_t current_call_id_;
 };
 } // namespace hbase

[08/50] [abbrv] hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells, addendum

[eclark]

HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells, addendum


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/356c1307
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/356c1307
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/356c1307

Branch: refs/heads/HBASE-14850
Commit: 356c1307021e931e871307077f71aab79dc43def
Parents: 0875c35
Author: Yu Li 
Authored: Mon Jul 11 22:43:26 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 22:44:32 2016 +0800

--
 .../hadoop/hbase/regionserver/AbstractMemStore.java   | 14 +++---
 .../hadoop/hbase/regionserver/MemStoreCompactor.java  |  4 ++--
 .../hadoop/hbase/regionserver/MutableSegment.java |  8 
 .../org/apache/hadoop/hbase/regionserver/Segment.java |  6 +++---
 4 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 79e95af..2b9910f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,8 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-boolean useMSLAB = (toAdd != cell);
-return internalAdd(toAdd, useMSLAB);
+boolean mslabUsed = (toAdd != cell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -157,8 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long delete(Cell deleteCell) {
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-boolean useMSLAB = (toAdd != deleteCell);
-long s = internalAdd(toAdd, useMSLAB);
+boolean mslabUsed = (toAdd != deleteCell);
+long s = internalAdd(toAdd, mslabUsed);
 return s;
   }
 
@@ -390,11 +390,11 @@ public abstract class AbstractMemStore implements 
MemStore {
*
* Callers should ensure they already have the read lock taken
* @param toAdd the cell to add
-   * @param useMSLAB whether using MSLAB
+   * @param mslabUsed whether using MSLAB
* @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
-long s = active.add(toAdd, useMSLAB);
+  private long internalAdd(final Cell toAdd, final boolean mslabUsed) {
+long s = active.add(toAdd, mslabUsed);
 setOldestEditTimeToNow();
 checkActiveSize();
 return s;

http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
index 23f792b..042de0a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
@@ -197,8 +197,8 @@ class MemStoreCompactor {
   // The scanner is doing all the elimination logic
   // now we just copy it to the new segment
   Cell newKV = result.maybeCloneWithAllocator(c);
-  boolean useMSLAB = (newKV != c);
-  result.internalAdd(newKV, useMSLAB);
+  boolean mslabUsed = (newKV != c);
+  result.internalAdd(newKV, mslabUsed);
 
 }
 kvs.clear();

http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index 6e54060..e62249a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -36,11 +36,11 @@ public class MutableSegment extends Segment {
   /**
* Adds the given cell into the segment
* @param cell the cell to add
-   * @param useMSLAB whether using MSLAB
+   * @param mslabUsed 

[19/50] [abbrv] hbase git commit: HBASE-14853 Add on protobuf to c++ chain

[eclark]

http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Master.proto
--
diff --git a/hbase-native-client/if/Master.proto 
b/hbase-native-client/if/Master.proto
new file mode 100644
index 000..4d3a2e1
--- /dev/null
+++ b/hbase-native-client/if/Master.proto
@@ -0,0 +1,778 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// All to do with the Master.  Includes schema management since these
+// changes are run by the Master process.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+import "ClusterStatus.proto";
+import "ErrorHandling.proto";
+import "Procedure.proto";
+import "Quota.proto";
+
+/* Column-level protobufs */
+
+message AddColumnRequest {
+  required TableName table_name = 1;
+  required ColumnFamilySchema column_families = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message AddColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DeleteColumnRequest {
+  required TableName table_name = 1;
+  required bytes column_name = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message DeleteColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyColumnRequest {
+  required TableName table_name = 1;
+  required ColumnFamilySchema column_families = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyColumnResponse {
+  optional uint64 proc_id = 1;
+}
+
+/* Region-level Protos */
+
+message MoveRegionRequest {
+  required RegionSpecifier region = 1;
+  optional ServerName dest_server_name = 2;
+}
+
+message MoveRegionResponse {
+}
+
+/**
+ * Dispatch merging the specified regions.
+ */
+message DispatchMergingRegionsRequest {
+  required RegionSpecifier region_a = 1;
+  required RegionSpecifier region_b = 2;
+  optional bool forcible = 3 [default = false];
+}
+
+message DispatchMergingRegionsResponse {
+}
+
+message AssignRegionRequest {
+  required RegionSpecifier region = 1;
+}
+
+message AssignRegionResponse {
+}
+
+message UnassignRegionRequest {
+  required RegionSpecifier region = 1;
+  optional bool force = 2 [default = false];
+}
+
+message UnassignRegionResponse {
+}
+
+message OfflineRegionRequest {
+  required RegionSpecifier region = 1;
+}
+
+message OfflineRegionResponse {
+}
+
+/* Table-level protobufs */
+
+message CreateTableRequest {
+  required TableSchema table_schema = 1;
+  repeated bytes split_keys = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message CreateTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DeleteTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message TruncateTableRequest {
+  required TableName tableName = 1;
+  optional bool preserveSplits = 2 [default = false];
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message TruncateTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message EnableTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message EnableTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message DisableTableRequest {
+  required TableName table_name = 1;
+  optional uint64 nonce_group = 2 [default = 0];
+  optional uint64 nonce = 3 [default = 0];
+}
+
+message DisableTableResponse {
+  optional uint64 proc_id = 1;
+}
+
+message ModifyTableRequest {
+  required TableName table_name = 1;
+  required TableSchema table_schema = 2;
+  optional uint64 nonce_group = 3 [default = 0];
+  optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyTableResponse 

[07/50] [abbrv] hbase git commit: HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)

[eclark]

HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0875c357
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0875c357
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0875c357

Branch: refs/heads/HBASE-14850
Commit: 0875c35781f1fcc06d38308b915a01791c7a6a20
Parents: 9cf012c
Author: tedyu 
Authored: Sun Jul 10 23:28:32 2016 -0700
Committer: tedyu 
Committed: Sun Jul 10 23:28:32 2016 -0700

--
 .../hadoop/hbase/replication/TestReplicationSmallTests.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0875c357/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 8efa67e..e3cc6e9 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -732,7 +732,9 @@ public class TestReplicationSmallTests extends 
TestReplicationBase {
 
   ClusterStatus status = admin.getClusterStatus();
 
-  for (ServerName server : status.getServers()) {
+  for (JVMClusterUtil.RegionServerThread thread :
+  utility1.getHBaseCluster().getRegionServerThreads()) {
+ServerName server = thread.getRegionServer().getServerName();
 ServerLoad sl = status.getLoad(server);
 List rLoadSourceList = 
sl.getReplicationLoadSourceList();
 ReplicationLoadSink rLoadSink = sl.getReplicationLoadSink();

[12/50] [abbrv] hbase git commit: HBASE-15766 Show working puts

[eclark]

HBASE-15766 Show working puts

Summary:
Add on showing how a set of puts to a single connection will work.
This still needs retries and looking up what region each request is going to

Test Plan:
./buck-out/gen/core/simple-client -columns 100
../bin/hbase shell
count 't'
100 row(s) in 0.2470 seconds

Differential Revision: https://reviews.facebook.net/D57603


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/79b5085d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/79b5085d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/79b5085d

Branch: refs/heads/HBASE-14850
Commit: 79b5085db5ca6283714ec5be2f5439d4d5d5eb72
Parents: f034294
Author: Elliott Clark 
Authored: Wed May 4 01:54:21 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 .../connection/client-dispatcher.cc |  9 +--
 .../connection/client-dispatcher.h  | 11 ++-
 .../connection/client-handler.cc| 13 ++--
 hbase-native-client/connection/client-handler.h |  4 +-
 .../connection/connection-factory.cc| 46 ++--
 .../connection/connection-factory.h | 11 ++-
 .../connection/connection-pool-test.cc  | 54 +++---
 .../connection/connection-pool.cc   | 33 +++--
 .../connection/connection-pool.h| 10 ++-
 hbase-native-client/core/client.cc  |  6 +-
 hbase-native-client/core/location-cache.cc  | 25 +++
 hbase-native-client/core/location-cache.h   |  6 +-
 hbase-native-client/core/meta-utils.cc  |  6 +-
 hbase-native-client/core/meta-utils.h   |  4 +-
 hbase-native-client/core/region-location.h  |  8 ++-
 hbase-native-client/core/simple-client.cc   | 76 +---
 .../serde/region-info-deserializer-test.cc  |  1 -
 hbase-native-client/serde/region-info.h |  6 +-
 18 files changed, 231 insertions(+), 98 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/client-dispatcher.cc
--
diff --git a/hbase-native-client/connection/client-dispatcher.cc 
b/hbase-native-client/connection/client-dispatcher.cc
index 6e2dc54..655d765 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -22,10 +22,11 @@ using namespace folly;
 using namespace hbase;
 using namespace wangle;
 
-ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
+ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
 
 void ClientDispatcher::read(Context *ctx, Response in) {
   auto call_id = in.call_id();
+
   auto search = requests_.find(call_id);
   CHECK(search != requests_.end());
   auto p = std::move(search->second);
@@ -38,10 +39,10 @@ void ClientDispatcher::read(Context *ctx, Response in) {
 }
 
 Future ClientDispatcher::operator()(std::unique_ptr arg) {
-  auto call_id = ++current_call_id_;
-
+  auto call_id = current_call_id_++;
   arg->set_call_id(call_id);
-  auto  = requests_[call_id];
+  requests_.insert(call_id, Promise{});
+  auto  = requests_.find(call_id)->second;
   auto f = p.getFuture();
   p.setInterruptHandler([call_id, this](const folly::exception_wrapper ) {
 LOG(ERROR) << "e = " << call_id;

http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/client-dispatcher.h
--
diff --git a/hbase-native-client/connection/client-dispatcher.h 
b/hbase-native-client/connection/client-dispatcher.h
index 826fc6a..4435a1b 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -19,8 +19,12 @@
 
 #pragma once
 
+#include 
+#include 
 #include 
 
+#include 
+
 #include "connection/pipeline.h"
 #include "connection/request.h"
 #include "connection/response.h"
@@ -31,13 +35,16 @@ class ClientDispatcher
   std::unique_ptr, Response> {
 public:
   ClientDispatcher();
+  ~ClientDispatcher() {
+LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
+  }
   void read(Context *ctx, Response in) override;
   folly::Future operator()(std::unique_ptr arg) override;
   folly::Future close(Context *ctx) override;
   folly::Future close() override;
 
 private:
-  std::unordered_map requests_;
+  folly::AtomicHashMap requests_;
   // Start at some number way above what could
   // be there for un-initialized call id counters.
   //
@@ -46,6 +53,6 @@ private:
   //
   // uint32_t has a max of 4Billion so 10 more 

[09/50] [abbrv] hbase git commit: HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)

[eclark]

HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b5fbf8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b5fbf8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b5fbf8d

Branch: refs/heads/HBASE-14850
Commit: 3b5fbf8d73b0ee50a0b04b044f93c1af39e5d79d
Parents: 356c130
Author: Gary Helmling 
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling 
Committed: Mon Jul 11 10:33:22 2016 -0700

--
 .../apache/hadoop/hbase/ipc/BufferChain.java|   5 +-
 .../hbase/security/token/SecureTestCluster.java | 123 +++
 .../TestDelegationTokenWithEncryption.java  |  91 ++
 .../token/TestGenerateDelegationToken.java  |  93 +-
 4 files changed, 218 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index 39efa40..7adc94d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -63,8 +63,9 @@ class BufferChain {
 byte [] bytes = new byte [this.remaining];
 int offset = 0;
 for (ByteBuffer bb: this.buffers) {
-  System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, 
bb.limit());
-  offset += bb.capacity();
+  int length = bb.remaining();
+  bb.get(bytes, offset, length);
+  offset += length;
 }
 return bytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 000..a469537
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+  protected static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+
+  protected static String USERNAME;
+
+  private static LocalHBaseCluster CLUSTER;
+
+  private static final File KEYTAB_FILE = new 
File(TEST_UTIL.getDataTestDir("keytab").toUri()
+  .getPath());
+  private static MiniKdc KDC;
+
+  private static String HOST = "localhost";
+
+  private static String PRINCIPAL;
+
+  private static String HTTP_PRINCIPAL;
+
+  /**
+   * Setup the security configuration for hdfs.
+   */
+  private static void setHdfsSecuredConfiguration(Configuration conf) throws 
Exception {
+// change XXX_USER_NAME_KEY to XXX_KERBEROS_PRINCIPAL_KEY after we 

[18/50] [abbrv] hbase git commit: HBASE-15078 Added ability to start/stop hbase local cluster for tests, global test_env for gtest, small changes to dockerfile and docker run.

[eclark]

HBASE-15078 Added ability to start/stop hbase local cluster for tests, global 
test_env for gtest, small changes to dockerfile and docker run.

Added check ~/.m2 folder exists; moved scripts to ./bin


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5431
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5431
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5431

Branch: refs/heads/HBASE-14850
Commit: 54314e3f1be9d6b5efcab845b8f321e4b4e2
Parents: 8556e3a
Author: Mikhail Antonov 
Authored: Wed Jan 6 15:08:21 2016 -0800
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/.buckconfig |  2 +-
 hbase-native-client/Dockerfile  |  2 +-
 hbase-native-client/bin/start-docker.sh | 10 -
 .../bin/start_local_hbase_and_wait.sh   | 26 
 .../bin/stop_local_hbase_and_wait.sh| 26 
 hbase-native-client/core/BUCK   | 15 +++
 .../core/HBaseNativeClientTestEnv.cc| 42 
 .../core/SampleNativeClientTest.cc  | 28 +
 hbase-native-client/core/test_env.h | 30 ++
 9 files changed, 178 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/.buckconfig
--
diff --git a/hbase-native-client/.buckconfig b/hbase-native-client/.buckconfig
index 3227a2a..402ef27 100644
--- a/hbase-native-client/.buckconfig
+++ b/hbase-native-client/.buckconfig
@@ -1,2 +1,2 @@
 [cxx]
-  gtest_dep = //third-party/googletest/googletest:google-test
+  gtest_dep = //third-party:google-test

http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/Dockerfile
--
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 70e823b..5f17f04 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -17,6 +17,6 @@
 
 FROM pjameson/buck-folly-watchman
 
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim 
maven inetutils-ping
 
 WORKDIR /usr/local/src/hbase/hbase-native-client

http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/bin/start-docker.sh
--
diff --git a/hbase-native-client/bin/start-docker.sh 
b/hbase-native-client/bin/start-docker.sh
index 1c9b02e..bf38912 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -28,5 +28,13 @@ if [[ ! -d third-party/googletest ]]; then
 git clone https://github.com/google/googletest.git 
third-party/googletest
 fi
 
+if [[ ! -d ~/.m2 ]]; then
+echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
+exit 1
+fi;
 
-docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native  /bin/bash
+docker run -p 16010:16010/tcp \
+   -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
+   -v ${PWD}/..:/usr/local/src/hbase \
+   -v ~/.m2:/root/.m2 \
+   -it hbase_native  /bin/bash

http://git-wip-us.apache.org/repos/asf/hbase/blob/5431/hbase-native-client/bin/start_local_hbase_and_wait.sh
--
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh 
b/hbase-native-client/bin/start_local_hbase_and_wait.sh
new file mode 100755
index 000..64d0b68
--- /dev/null
+++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$PWD/../bin/start-hbase.sh
+
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == 
"200" ]
+do
+ printf "Waiting for local HBase 

[23/50] [abbrv] hbase git commit: HBASE-15649 Ignore buck-out when building docker image

[eclark]

HBASE-15649 Ignore buck-out when building docker image


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/098028c5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/098028c5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/098028c5

Branch: refs/heads/HBASE-14850
Commit: 098028c5c61ae7d318dde73fece2c0758f28d113
Parents: bfabc71
Author: Elliott Clark 
Authored: Wed Apr 13 17:02:38 2016 -0700
Committer: Elliott Clark 
Committed: Mon Jul 11 16:47:26 2016 -0700

--
 hbase-native-client/.dockerignore | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/098028c5/hbase-native-client/.dockerignore
--
diff --git a/hbase-native-client/.dockerignore 
b/hbase-native-client/.dockerignore
new file mode 100644
index 000..2cfffb7
--- /dev/null
+++ b/hbase-native-client/.dockerignore
@@ -0,0 +1,2 @@
+buck-out
+.buckd

[03/50] [abbrv] hbase git commit: HBASE-16201 NPE in RpcServer causing intermittent UT failure of TestMasterReplication#testHFileCyclicReplication

[eclark]

HBASE-16201 NPE in RpcServer causing intermittent UT failure of 
TestMasterReplication#testHFileCyclicReplication


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3c39cbd9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3c39cbd9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3c39cbd9

Branch: refs/heads/HBASE-14850
Commit: 3c39cbd92c3f309c98ca01bfb70ca89bc046a228
Parents: 1578a04
Author: Yu Li 
Authored: Sat Jul 9 01:05:10 2016 +0800
Committer: Yu Li 
Committed: Sat Jul 9 02:02:00 2016 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3c39cbd9/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index ad88e8e..73226aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -2246,7 +2246,13 @@ public class RpcServer implements RpcServerInterface, 
ConfigurationObserver {
   // The above callBlockingMethod will always return a SE.  Strip the SE 
wrapper before
   // putting it on the wire.  Its needed to adhere to the pb Service 
Interface but we don't
   // need to pass it over the wire.
-  if (e instanceof ServiceException) e = e.getCause();
+  if (e instanceof ServiceException) {
+if (e.getCause() == null) {
+  LOG.debug("Caught a ServiceException with null cause", e);
+} else {
+  e = e.getCause();
+}
+  }
 
   // increment the number of requests that were exceptions.
   metrics.exception(e);

[02/50] [abbrv] hbase git commit: Revert "NPE in RpcServer causing intermittent UT failure of TestMasterReplication#testHFileCyclicReplication"

[eclark]

Revert "NPE in RpcServer causing intermittent UT failure of 
TestMasterReplication#testHFileCyclicReplication"

This reverts commit a33097e067b73be8e877b822afa90b89a0c7974f.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1578a045
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1578a045
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1578a045

Branch: refs/heads/HBASE-14850
Commit: 1578a045b6af6e8cd4c85d7b43397e204345a99d
Parents: 5e0d97e
Author: Yu Li 
Authored: Sat Jul 9 02:01:43 2016 +0800
Committer: Yu Li 
Committed: Sat Jul 9 02:01:43 2016 +0800

--
 .../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 8 +---
 1 file changed, 1 insertion(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/1578a045/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 73226aa..ad88e8e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -2246,13 +2246,7 @@ public class RpcServer implements RpcServerInterface, 
ConfigurationObserver {
   // The above callBlockingMethod will always return a SE.  Strip the SE 
wrapper before
   // putting it on the wire.  Its needed to adhere to the pb Service 
Interface but we don't
   // need to pass it over the wire.
-  if (e instanceof ServiceException) {
-if (e.getCause() == null) {
-  LOG.debug("Caught a ServiceException with null cause", e);
-} else {
-  e = e.getCause();
-}
-  }
+  if (e instanceof ServiceException) e = e.getCause();
 
   // increment the number of requests that were exceptions.
   metrics.exception(e);

hbase git commit: HBASE-11625 - Verifies data before building HFileBlock. - Adds HFileBlock.Header class which contains information about location of fields. Testing: Adds CorruptedFSReaderImpl to Tes

[appy]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 25c7dee21 -> 79b77e354


HBASE-11625 - Verifies data before building HFileBlock. - Adds 
HFileBlock.Header class which contains information about location of fields. 
Testing: Adds CorruptedFSReaderImpl to TestChecksum.

Change-Id: I107e9636b28abb6b15ec329e885f1e31b1b1b988

Signed-off-by: stack 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/79b77e35
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/79b77e35
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/79b77e35

Branch: refs/heads/branch-1.1
Commit: 79b77e3542a93d362c5565c98ce8d8e1f0044337
Parents: 25c7dee
Author: Apekshit 
Authored: Thu May 5 17:05:17 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Jul 11 11:39:48 2016 -0700

--
 .../hadoop/hbase/io/hfile/ChecksumUtil.java |  80 
 .../hadoop/hbase/io/hfile/HFileBlock.java   | 187 ++-
 .../hadoop/hbase/io/hfile/TestChecksum.java |  60 --
 3 files changed, 190 insertions(+), 137 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/79b77e35/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
index 0e03a42..ecf5f00 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
@@ -22,11 +22,13 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.zip.Checksum;
 
+import org.apache.hadoop.fs.ChecksumException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.util.ByteBufferUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ChecksumType;
+import org.apache.hadoop.util.DataChecksum;
 
 /**
  * Utility methods to compute and validate checksums.
@@ -90,41 +92,37 @@ public class ChecksumUtil {
   }
 
   /**
-   * Validates that the data in the specified HFileBlock matches the
-   * checksum.  Generates the checksum for the data and
-   * then validate that it matches the value stored in the header.
-   * If there is a checksum mismatch, then return false. Otherwise
-   * return true.
-   * The header is extracted from the specified HFileBlock while the
-   * data-to-be-verified is extracted from 'data'.
+   * Validates that the data in the specified HFileBlock matches the checksum. 
Generates the
+   * checksums for the data and then validate that it matches those stored in 
the end of the data.
+   * @param buffer Contains the data in following order: HFileBlock header, 
data, checksums.
+   * @param path Path of the HFile to which the {@code data} belongs. Only 
used for logging.
+   * @param offset offset of the data being validated. Only used for logging.
+   * @param hdrSize Size of the block header in {@code data}. Only used for 
logging.
+   * @return True if checksum matches, else false.
*/
-  static boolean validateBlockChecksum(Path path, HFileBlock block, 
-byte[] data, int hdrSize) throws IOException {
-
-// If this is an older version of the block that does not have
-// checksums, then return false indicating that checksum verification
-// did not succeed. Actually, this methiod should never be called
-// when the minorVersion is 0, thus this is a defensive check for a
-// cannot-happen case. Since this is a cannot-happen case, it is
-// better to return false to indicate a checksum validation failure.
-if (!block.getHFileContext().isUseHBaseChecksum()) {
-  return false;
-}
-
-// Get a checksum object based on the type of checksum that is
-// set in the HFileBlock header. A ChecksumType.NULL indicates that 
-// the caller is not interested in validating checksums, so we
-// always return true.
-ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType());
+  static boolean validateChecksum(ByteBuffer buffer, Path path, long offset, 
int hdrSize)
+  throws IOException {
+// A ChecksumType.NULL indicates that the caller is not interested in 
validating checksums,
+// so we always return true.
+ChecksumType cktype =
+
ChecksumType.codeToType(buffer.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
 if (cktype == ChecksumType.NULL) {
   return true; // No checkums validations needed for this block.
 }
-Checksum checksumObject = cktype.getChecksumObject();
-

hbase git commit: HBASE-16044 Fix 'hbase shell' output parsing in graceful_stop.sh

[appy]

Repository: hbase
Updated Branches:
  refs/heads/master 3b5fbf8d7 -> a396ae773


HBASE-16044 Fix 'hbase shell' output parsing in graceful_stop.sh

Change-Id: I94b2f665105837d31281be66d620b6152c3bc9d1


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a396ae77
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a396ae77
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a396ae77

Branch: refs/heads/master
Commit: a396ae773a0f7dbcd6f54fc8133b7a9dc30d8431
Parents: 3b5fbf8
Author: Apekshit Sharma 
Authored: Tue Jun 28 16:55:09 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Jul 11 11:28:33 2016 -0700

--
 bin/graceful_stop.sh | 2 +-
 bin/rolling-restart.sh   | 2 +-
 hbase-shell/src/main/ruby/shell/commands/balance_switch.rb   | 4 +++-
 hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb | 4 +++-
 4 files changed, 8 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/bin/graceful_stop.sh
--
diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh
index 8d08cc9..89e3dd9 100755
--- a/bin/graceful_stop.sh
+++ b/bin/graceful_stop.sh
@@ -106,7 +106,7 @@ if [ "$nob" == "true"  ]; then
   HBASE_BALANCER_STATE=false
 else
   log "Disabling load balancer"
-  HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config 
"${HBASE_CONF_DIR}" shell | tail -3 | head -1)
+  HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config 
"${HBASE_CONF_DIR}" shell -n | tail -1)
   log "Previous balancer state was $HBASE_BALANCER_STATE"
 fi
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/bin/rolling-restart.sh
--
diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 530b76b..9ac4d64 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -189,7 +189,7 @@ else
 zkrs="$zparent/$zkrs"
 online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed 
"s/\[//" | sed "s/\]//"`
 echo "Disabling load balancer"
-HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config 
"${HBASE_CONF_DIR}" shell | tail -3 | head -1)
+HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config 
"${HBASE_CONF_DIR}" shell -n | tail -1)
 echo "Previous balancer state was $HBASE_BALANCER_STATE"
 
 for rs in $online_regionservers

http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb 
b/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
index 107d267..4076cf6 100644
--- a/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
@@ -31,7 +31,9 @@ EOF
   end
 
   def command(enableDisable)
-formatter.row([admin.balance_switch(enableDisable)? "true" : "false"])
+prev_state = admin.balance_switch(enableDisable)? "true" : "false"
+formatter.row(["Previous balancer state : #{prev_state}"])
+prev_state
   end
 end
   end

http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
--
diff --git a/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb 
b/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
index 6a75ab5..c3d7a2b 100644
--- a/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
@@ -30,7 +30,9 @@ EOF
   end
 
   def command()
-formatter.row([admin.balancer_enabled?.to_s])
+state = admin.balancer_enabled?
+formatter.row([state.to_s])
+state
   end
 end
   end

hbase git commit: HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)

[garyh]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 93b00b2aa -> 00c91b01e


HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/00c91b01
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/00c91b01
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/00c91b01

Branch: refs/heads/branch-1.3
Commit: 00c91b01ea0d89810541e0bef3152605519f9f70
Parents: 93b00b2
Author: Gary Helmling 
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling 
Committed: Mon Jul 11 10:45:51 2016 -0700

--
 .../apache/hadoop/hbase/ipc/BufferChain.java|   5 +-
 .../hbase/security/token/SecureTestCluster.java | 123 +++
 .../TestDelegationTokenWithEncryption.java  |  91 ++
 .../token/TestGenerateDelegationToken.java  |  96 +--
 4 files changed, 220 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/00c91b01/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index babd2f8..1a8fa5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -56,8 +56,9 @@ class BufferChain {
 byte [] bytes = new byte [this.remaining];
 int offset = 0;
 for (ByteBuffer bb: this.buffers) {
-  System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, 
bb.limit());
-  offset += bb.capacity();
+  int length = bb.remaining();
+  bb.get(bytes, offset, length);
+  offset += length;
 }
 return bytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/00c91b01/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 000..a469537
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+  protected static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+
+  protected static String USERNAME;
+
+  private static LocalHBaseCluster CLUSTER;
+
+  private static final File KEYTAB_FILE = new 
File(TEST_UTIL.getDataTestDir("keytab").toUri()
+  .getPath());
+  private static MiniKdc KDC;
+
+  private static String HOST = "localhost";
+
+  private static String PRINCIPAL;
+
+  private static String HTTP_PRINCIPAL;
+
+  /**
+   * Setup the security configuration for hdfs.
+   */
+  private static void setHdfsSecuredConfiguration(Configuration conf) throws 

hbase git commit: HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)

[garyh]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 e03d4fbb0 -> 3ff6f4585


HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3ff6f458
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3ff6f458
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3ff6f458

Branch: refs/heads/branch-1
Commit: 3ff6f4585104101e0a85666a09376ed4b3ad790b
Parents: e03d4fb
Author: Gary Helmling 
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling 
Committed: Mon Jul 11 10:45:05 2016 -0700

--
 .../apache/hadoop/hbase/ipc/BufferChain.java|   5 +-
 .../hbase/security/token/SecureTestCluster.java | 123 +++
 .../TestDelegationTokenWithEncryption.java  |  91 ++
 .../token/TestGenerateDelegationToken.java  |  96 +--
 4 files changed, 220 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3ff6f458/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index babd2f8..1a8fa5b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -56,8 +56,9 @@ class BufferChain {
 byte [] bytes = new byte [this.remaining];
 int offset = 0;
 for (ByteBuffer bb: this.buffers) {
-  System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, 
bb.limit());
-  offset += bb.capacity();
+  int length = bb.remaining();
+  bb.get(bytes, offset, length);
+  offset += length;
 }
 return bytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3ff6f458/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 000..a469537
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+  protected static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+
+  protected static String USERNAME;
+
+  private static LocalHBaseCluster CLUSTER;
+
+  private static final File KEYTAB_FILE = new 
File(TEST_UTIL.getDataTestDir("keytab").toUri()
+  .getPath());
+  private static MiniKdc KDC;
+
+  private static String HOST = "localhost";
+
+  private static String PRINCIPAL;
+
+  private static String HTTP_PRINCIPAL;
+
+  /**
+   * Setup the security configuration for hdfs.
+   */
+  private static void setHdfsSecuredConfiguration(Configuration conf) throws 

hbase git commit: HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)

[garyh]

Repository: hbase
Updated Branches:
  refs/heads/master 356c13070 -> 3b5fbf8d7


HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b5fbf8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b5fbf8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b5fbf8d

Branch: refs/heads/master
Commit: 3b5fbf8d73b0ee50a0b04b044f93c1af39e5d79d
Parents: 356c130
Author: Gary Helmling 
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling 
Committed: Mon Jul 11 10:33:22 2016 -0700

--
 .../apache/hadoop/hbase/ipc/BufferChain.java|   5 +-
 .../hbase/security/token/SecureTestCluster.java | 123 +++
 .../TestDelegationTokenWithEncryption.java  |  91 ++
 .../token/TestGenerateDelegationToken.java  |  93 +-
 4 files changed, 218 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index 39efa40..7adc94d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -63,8 +63,9 @@ class BufferChain {
 byte [] bytes = new byte [this.remaining];
 int offset = 0;
 for (ByteBuffer bb: this.buffers) {
-  System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, 
bb.limit());
-  offset += bb.capacity();
+  int length = bb.remaining();
+  bb.get(bytes, offset, length);
+  offset += length;
 }
 return bytes;
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 000..a469537
--- /dev/null
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+  protected static final HBaseTestingUtility TEST_UTIL = new 
HBaseTestingUtility();
+
+  protected static String USERNAME;
+
+  private static LocalHBaseCluster CLUSTER;
+
+  private static final File KEYTAB_FILE = new 
File(TEST_UTIL.getDataTestDir("keytab").toUri()
+  .getPath());
+  private static MiniKdc KDC;
+
+  private static String HOST = "localhost";
+
+  private static String PRINCIPAL;
+
+  private static String HTTP_PRINCIPAL;
+
+  /**
+   * Setup the security configuration for hdfs.
+   */
+  private static void setHdfsSecuredConfiguration(Configuration conf) throws 
Exception {

hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells, addendum

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/master 0875c3578 -> 356c13070


HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells, addendum


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/356c1307
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/356c1307
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/356c1307

Branch: refs/heads/master
Commit: 356c1307021e931e871307077f71aab79dc43def
Parents: 0875c35
Author: Yu Li 
Authored: Mon Jul 11 22:43:26 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 22:44:32 2016 +0800

--
 .../hadoop/hbase/regionserver/AbstractMemStore.java   | 14 +++---
 .../hadoop/hbase/regionserver/MemStoreCompactor.java  |  4 ++--
 .../hadoop/hbase/regionserver/MutableSegment.java |  8 
 .../org/apache/hadoop/hbase/regionserver/Segment.java |  6 +++---
 4 files changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 79e95af..2b9910f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,8 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-boolean useMSLAB = (toAdd != cell);
-return internalAdd(toAdd, useMSLAB);
+boolean mslabUsed = (toAdd != cell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -157,8 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
   @Override
   public long delete(Cell deleteCell) {
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-boolean useMSLAB = (toAdd != deleteCell);
-long s = internalAdd(toAdd, useMSLAB);
+boolean mslabUsed = (toAdd != deleteCell);
+long s = internalAdd(toAdd, mslabUsed);
 return s;
   }
 
@@ -390,11 +390,11 @@ public abstract class AbstractMemStore implements 
MemStore {
*
* Callers should ensure they already have the read lock taken
* @param toAdd the cell to add
-   * @param useMSLAB whether using MSLAB
+   * @param mslabUsed whether using MSLAB
* @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
-long s = active.add(toAdd, useMSLAB);
+  private long internalAdd(final Cell toAdd, final boolean mslabUsed) {
+long s = active.add(toAdd, mslabUsed);
 setOldestEditTimeToNow();
 checkActiveSize();
 return s;

http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
index 23f792b..042de0a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
@@ -197,8 +197,8 @@ class MemStoreCompactor {
   // The scanner is doing all the elimination logic
   // now we just copy it to the new segment
   Cell newKV = result.maybeCloneWithAllocator(c);
-  boolean useMSLAB = (newKV != c);
-  result.internalAdd(newKV, useMSLAB);
+  boolean mslabUsed = (newKV != c);
+  result.internalAdd(newKV, mslabUsed);
 
 }
 kvs.clear();

http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index 6e54060..e62249a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -36,11 +36,11 @@ public class MutableSegment extends Segment {
   /**
* Adds the given cell into the segment
* @param cell the cell 

hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 73189eb80 -> 25c7dee21


HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/25c7dee2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/25c7dee2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/25c7dee2

Branch: refs/heads/branch-1.1
Commit: 25c7dee21e7060f3be586b05539ae5954c0a
Parents: 73189eb
Author: Yu Li 
Authored: Sun Jul 10 14:09:02 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 21:46:55 2016 +0800

--
 .../hbase/regionserver/DefaultMemStore.java | 38 ++--
 .../hbase/regionserver/HeapMemStoreLAB.java | 11 ++
 .../hbase/regionserver/TestDefaultMemStore.java | 21 +++
 3 files changed, 60 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/25c7dee2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 3da0c0b..7787b23 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.htrace.Trace;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * The MemStore holds in-memory modifications to the Store.  Modifications
  * are {@link Cell}s.  When asked to flush, current memstore is moved
@@ -223,7 +225,8 @@ public class DefaultMemStore implements MemStore {
   @Override
   public Pair add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-return new Pair(internalAdd(toAdd), toAdd);
+boolean mslabUsed = (toAdd != cell);
+return new Pair(internalAdd(toAdd, mslabUsed), toAdd);
   }
 
   @Override
@@ -254,20 +257,38 @@ public class DefaultMemStore implements MemStore {
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+   * @param toAdd the cell to add
+   * @param mslabUsed whether using MSLAB
+   * @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd) {
-long s = heapSizeChange(toAdd, addToCellSet(toAdd));
+  private long internalAdd(final Cell toAdd, boolean mslabUsed) {
+boolean notPresent = addToCellSet(toAdd);
+long s = heapSizeChange(toAdd, notPresent);
+// If there's already a same cell in the CellSet and we are using MSLAB, 
we must count in the
+// MSLAB allocation size as well, or else there will be memory leak 
(occupied heap size larger
+// than the counted number)
+if (!notPresent && mslabUsed) {
+  s += getCellLength(toAdd);
+}
 timeRangeTracker.includeTimestamp(toAdd);
 this.size.addAndGet(s);
 return s;
   }
 
+  /**
+   * Get cell length after serialized in {@link KeyValue}
+   */
+  @VisibleForTesting
+  int getCellLength(Cell cell) {
+return KeyValueUtil.length(cell);
+  }
+
   private Cell maybeCloneWithAllocator(Cell cell) {
 if (allocator == null) {
   return cell;
 }
 
-int len = KeyValueUtil.length(cell);
+int len = getCellLength(cell);
 ByteRange alloc = allocator.allocateBytes(len);
 if (alloc == null) {
   // The allocation was too large, allocator decided
@@ -318,12 +339,9 @@ public class DefaultMemStore implements MemStore {
*/
   @Override
   public long delete(Cell deleteCell) {
-long s = 0;
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-s += heapSizeChange(toAdd, addToCellSet(toAdd));
-timeRangeTracker.includeTimestamp(toAdd);
-this.size.addAndGet(s);
-return s;
+boolean mslabUsed = (toAdd != deleteCell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -564,7 +582,7 @@ public class DefaultMemStore implements MemStore {
 // hitting OOME - see TestMemStore.testUpsertMSLAB for a
 // test that triggers the pathological case if we don't avoid MSLAB
 // here.
-long addedSize = internalAdd(cell);
+long addedSize = internalAdd(cell, false);
 
 // Get the Cells for the row/family/qualifier regardless of timestamp.
 // For this case we want to clean up any other puts

http://git-wip-us.apache.org/repos/asf/hbase/blob/25c7dee2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java

hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 b99efe65e -> b96b04204


HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b96b0420
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b96b0420
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b96b0420

Branch: refs/heads/branch-1.2
Commit: b96b042043d96cbb348b538a7abc9d44157948b9
Parents: b99efe6
Author: Yu Li 
Authored: Sun Jul 10 14:09:02 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 21:49:44 2016 +0800

--
 .../hbase/regionserver/DefaultMemStore.java | 38 ++--
 .../hbase/regionserver/HeapMemStoreLAB.java | 11 ++
 .../hbase/regionserver/TestDefaultMemStore.java | 21 +++
 3 files changed, 60 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b96b0420/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index c929e71..f8fc397 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.htrace.Trace;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * The MemStore holds in-memory modifications to the Store.  Modifications
  * are {@link Cell}s.  When asked to flush, current memstore is moved
@@ -226,7 +228,8 @@ public class DefaultMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-return internalAdd(toAdd);
+boolean mslabUsed = (toAdd != cell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   @Override
@@ -264,20 +267,38 @@ public class DefaultMemStore implements MemStore {
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+   * @param toAdd the cell to add
+   * @param mslabUsed whether using MSLAB
+   * @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd) {
-long s = heapSizeChange(toAdd, addToCellSet(toAdd));
+  private long internalAdd(final Cell toAdd, boolean mslabUsed) {
+boolean notPresent = addToCellSet(toAdd);
+long s = heapSizeChange(toAdd, notPresent);
+// If there's already a same cell in the CellSet and we are using MSLAB, 
we must count in the
+// MSLAB allocation size as well, or else there will be memory leak 
(occupied heap size larger
+// than the counted number)
+if (!notPresent && mslabUsed) {
+  s += getCellLength(toAdd);
+}
 timeRangeTracker.includeTimestamp(toAdd);
 this.size.addAndGet(s);
 return s;
   }
 
+  /**
+   * Get cell length after serialized in {@link KeyValue}
+   */
+  @VisibleForTesting
+  int getCellLength(Cell cell) {
+return KeyValueUtil.length(cell);
+  }
+
   private Cell maybeCloneWithAllocator(Cell cell) {
 if (allocator == null) {
   return cell;
 }
 
-int len = KeyValueUtil.length(cell);
+int len = getCellLength(cell);
 ByteRange alloc = allocator.allocateBytes(len);
 if (alloc == null) {
   // The allocation was too large, allocator decided
@@ -328,12 +349,9 @@ public class DefaultMemStore implements MemStore {
*/
   @Override
   public long delete(Cell deleteCell) {
-long s = 0;
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-s += heapSizeChange(toAdd, addToCellSet(toAdd));
-timeRangeTracker.includeTimestamp(toAdd);
-this.size.addAndGet(s);
-return s;
+boolean mslabUsed = (toAdd != deleteCell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -574,7 +592,7 @@ public class DefaultMemStore implements MemStore {
 // hitting OOME - see TestMemStore.testUpsertMSLAB for a
 // test that triggers the pathological case if we don't avoid MSLAB
 // here.
-long addedSize = internalAdd(cell);
+long addedSize = internalAdd(cell, false);
 
 // Get the Cells for the row/family/qualifier regardless of timestamp.
 // For this case we want to clean up any other puts

http://git-wip-us.apache.org/repos/asf/hbase/blob/b96b0420/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java

hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 e7513e447 -> 93b00b2aa


HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/93b00b2a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/93b00b2a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/93b00b2a

Branch: refs/heads/branch-1.3
Commit: 93b00b2aae2cca7229637398593441b768f5debb
Parents: e7513e4
Author: Yu Li 
Authored: Sun Jul 10 14:09:02 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 21:50:10 2016 +0800

--
 .../hbase/regionserver/DefaultMemStore.java | 38 ++--
 .../hbase/regionserver/HeapMemStoreLAB.java | 11 ++
 .../hbase/regionserver/TestDefaultMemStore.java | 21 +++
 3 files changed, 60 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/93b00b2a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 05041f5..458e3b8 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.htrace.Trace;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * The MemStore holds in-memory modifications to the Store.  Modifications
  * are {@link Cell}s.  When asked to flush, current memstore is moved
@@ -226,7 +228,8 @@ public class DefaultMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-return internalAdd(toAdd);
+boolean mslabUsed = (toAdd != cell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   @Override
@@ -264,20 +267,38 @@ public class DefaultMemStore implements MemStore {
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+   * @param toAdd the cell to add
+   * @param mslabUsed whether using MSLAB
+   * @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd) {
-long s = heapSizeChange(toAdd, addToCellSet(toAdd));
+  private long internalAdd(final Cell toAdd, boolean mslabUsed) {
+boolean notPresent = addToCellSet(toAdd);
+long s = heapSizeChange(toAdd, notPresent);
+// If there's already a same cell in the CellSet and we are using MSLAB, 
we must count in the
+// MSLAB allocation size as well, or else there will be memory leak 
(occupied heap size larger
+// than the counted number)
+if (!notPresent && mslabUsed) {
+  s += getCellLength(toAdd);
+}
 timeRangeTracker.includeTimestamp(toAdd);
 this.size.addAndGet(s);
 return s;
   }
 
+  /**
+   * Get cell length after serialized in {@link KeyValue}
+   */
+  @VisibleForTesting
+  int getCellLength(Cell cell) {
+return KeyValueUtil.length(cell);
+  }
+
   private Cell maybeCloneWithAllocator(Cell cell) {
 if (allocator == null) {
   return cell;
 }
 
-int len = KeyValueUtil.length(cell);
+int len = getCellLength(cell);
 ByteRange alloc = allocator.allocateBytes(len);
 if (alloc == null) {
   // The allocation was too large, allocator decided
@@ -328,12 +349,9 @@ public class DefaultMemStore implements MemStore {
*/
   @Override
   public long delete(Cell deleteCell) {
-long s = 0;
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-s += heapSizeChange(toAdd, addToCellSet(toAdd));
-timeRangeTracker.includeTimestamp(toAdd);
-this.size.addAndGet(s);
-return s;
+boolean mslabUsed = (toAdd != deleteCell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -574,7 +592,7 @@ public class DefaultMemStore implements MemStore {
 // hitting OOME - see TestMemStore.testUpsertMSLAB for a
 // test that triggers the pathological case if we don't avoid MSLAB
 // here.
-long addedSize = internalAdd(cell);
+long addedSize = internalAdd(cell, false);
 
 // Get the Cells for the row/family/qualifier regardless of timestamp.
 // For this case we want to clean up any other puts

http://git-wip-us.apache.org/repos/asf/hbase/blob/93b00b2a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java

hbase git commit: HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells

[liyu]

Repository: hbase
Updated Branches:
  refs/heads/branch-1 fe915e10d -> e03d4fbb0


HBASE-16194 Should count in MSLAB chunk allocation into heap size change when 
adding duplicate cells


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e03d4fbb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e03d4fbb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e03d4fbb

Branch: refs/heads/branch-1
Commit: e03d4fbb0a90483a8a72e0682b02f387ba4b7427
Parents: fe915e1
Author: Yu Li 
Authored: Sun Jul 10 14:09:02 2016 +0800
Committer: Yu Li 
Committed: Mon Jul 11 21:40:30 2016 +0800

--
 .../hbase/regionserver/DefaultMemStore.java | 38 ++--
 .../hbase/regionserver/HeapMemStoreLAB.java | 11 ++
 .../hbase/regionserver/TestDefaultMemStore.java | 21 +++
 3 files changed, 60 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e03d4fbb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index ce793a9..71bfca0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 import org.apache.htrace.Trace;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * The MemStore holds in-memory modifications to the Store.  Modifications
  * are {@link Cell}s.  When asked to flush, current memstore is moved
@@ -226,7 +228,8 @@ public class DefaultMemStore implements MemStore {
   @Override
   public long add(Cell cell) {
 Cell toAdd = maybeCloneWithAllocator(cell);
-return internalAdd(toAdd);
+boolean mslabUsed = (toAdd != cell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   @Override
@@ -264,20 +267,38 @@ public class DefaultMemStore implements MemStore {
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+   * @param toAdd the cell to add
+   * @param mslabUsed whether using MSLAB
+   * @return the heap size change in bytes
*/
-  private long internalAdd(final Cell toAdd) {
-long s = heapSizeChange(toAdd, addToCellSet(toAdd));
+  private long internalAdd(final Cell toAdd, boolean mslabUsed) {
+boolean notPresent = addToCellSet(toAdd);
+long s = heapSizeChange(toAdd, notPresent);
+// If there's already a same cell in the CellSet and we are using MSLAB, 
we must count in the
+// MSLAB allocation size as well, or else there will be memory leak 
(occupied heap size larger
+// than the counted number)
+if (!notPresent && mslabUsed) {
+  s += getCellLength(toAdd);
+}
 timeRangeTracker.includeTimestamp(toAdd);
 this.size.addAndGet(s);
 return s;
   }
 
+  /**
+   * Get cell length after serialized in {@link KeyValue}
+   */
+  @VisibleForTesting
+  int getCellLength(Cell cell) {
+return KeyValueUtil.length(cell);
+  }
+
   private Cell maybeCloneWithAllocator(Cell cell) {
 if (allocator == null) {
   return cell;
 }
 
-int len = KeyValueUtil.length(cell);
+int len = getCellLength(cell);
 ByteRange alloc = allocator.allocateBytes(len);
 if (alloc == null) {
   // The allocation was too large, allocator decided
@@ -328,12 +349,9 @@ public class DefaultMemStore implements MemStore {
*/
   @Override
   public long delete(Cell deleteCell) {
-long s = 0;
 Cell toAdd = maybeCloneWithAllocator(deleteCell);
-s += heapSizeChange(toAdd, addToCellSet(toAdd));
-timeRangeTracker.includeTimestamp(toAdd);
-this.size.addAndGet(s);
-return s;
+boolean mslabUsed = (toAdd != deleteCell);
+return internalAdd(toAdd, mslabUsed);
   }
 
   /**
@@ -574,7 +592,7 @@ public class DefaultMemStore implements MemStore {
 // hitting OOME - see TestMemStore.testUpsertMSLAB for a
 // test that triggers the pathological case if we don't avoid MSLAB
 // here.
-long addedSize = internalAdd(cell);
+long addedSize = internalAdd(cell, false);
 
 // Get the Cells for the row/family/qualifier regardless of timestamp.
 // For this case we want to clean up any other puts

http://git-wip-us.apache.org/repos/asf/hbase/blob/e03d4fbb/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java

hbase git commit: HBASE-16055 PutSortReducer loses any Visibility/acl attribute set on the Puts (Ram)

[ramkrishna]

Repository: hbase
Updated Branches:
  refs/heads/0.98 60ff6128e -> ecc1a886e


HBASE-16055 PutSortReducer loses any Visibility/acl attribute set on the
Puts (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ecc1a886
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ecc1a886
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ecc1a886

Branch: refs/heads/0.98
Commit: ecc1a886e4dd8747595a82d17eb805eabf26c20b
Parents: 60ff612
Author: Ramkrishna 
Authored: Mon Jul 11 12:17:31 2016 +0530
Committer: Ramkrishna 
Committed: Mon Jul 11 12:17:31 2016 +0530

--
 .../DefaultVisibilityExpressionResolver.java|   7 +-
 .../hadoop/hbase/mapreduce/PutSortReducer.java  |  61 +++-
 .../hadoop/hbase/mapreduce/TextSortReducer.java |   8 +-
 .../hbase/mapreduce/TestHFileOutputFormat2.java | 147 ++-
 4 files changed, 179 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ecc1a886/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
index 1954f68..39d6898 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/DefaultVisibilityExpressionResolver.java
@@ -70,10 +70,6 @@ public class DefaultVisibilityExpressionResolver implements 
VisibilityExpression
 HTable labelsTable = null;
 try {
   labelsTable = new HTable(conf, LABELS_TABLE_NAME);
-} catch (TableNotFoundException e) {
-  // Just return with out doing any thing. When the VC is not used we wont 
be having 'labels'
-  // table in the cluster.
-  return;
 } catch (IOException e) {
   LOG.error("Error opening 'labels' table", e);
   return;
@@ -90,6 +86,9 @@ public class DefaultVisibilityExpressionResolver implements 
VisibilityExpression
 byte[] value = next.getValue(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
 labels.put(Bytes.toString(value), Bytes.toInt(row));
   }
+} catch (TableNotFoundException e) {
+  // Table not found. So just return
+  return;
 } catch (IOException e) {
   LOG.error("Error reading 'labels' table", e);
 } finally {

http://git-wip-us.apache.org/repos/asf/hbase/blob/ecc1a886/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index 792686a..4160cdd 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -18,17 +18,25 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.TreeSet;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.TagType;
 import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.security.visibility.CellVisibility;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.util.StringUtils;
 
@@ -44,7 +52,17 @@ import org.apache.hadoop.util.StringUtils;
 @InterfaceStability.Stable
 public class PutSortReducer extends
 Reducer {
-  
+  // the cell creator
+  private CellCreator kvCreator;
+
+  @Override
+  protected void
+  setup(Reducer.Context context)
+  throws IOException, InterruptedException {
+Configuration conf = context.getConfiguration();
+this.kvCreator = new CellCreator(conf);
+  }
+
   @Override
   protected void reduce(
   ImmutableBytesWritable 

hbase git commit: HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/master 9cf012cd0 -> 0875c3578


HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0875c357
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0875c357
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0875c357

Branch: refs/heads/master
Commit: 0875c35781f1fcc06d38308b915a01791c7a6a20
Parents: 9cf012c
Author: tedyu 
Authored: Sun Jul 10 23:28:32 2016 -0700
Committer: tedyu 
Committed: Sun Jul 10 23:28:32 2016 -0700

--
 .../hadoop/hbase/replication/TestReplicationSmallTests.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0875c357/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 8efa67e..e3cc6e9 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -732,7 +732,9 @@ public class TestReplicationSmallTests extends 
TestReplicationBase {
 
   ClusterStatus status = admin.getClusterStatus();
 
-  for (ServerName server : status.getServers()) {
+  for (JVMClusterUtil.RegionServerThread thread :
+  utility1.getHBaseCluster().getRegionServerThreads()) {
+ServerName server = thread.getRegionServer().getServerName();
 ServerLoad sl = status.getLoad(server);
 List rLoadSourceList = 
sl.getReplicationLoadSourceList();
 ReplicationLoadSink rLoadSink = sl.getReplicationLoadSink();