kudu git commit: [Java] Fix TestSecurity tests on Mac

2018-05-03 Thread granthenke 
Repository: kudu
Updated Branches:
  refs/heads/branch-1.7.x ffc01ab72 -> fb4bc2f64


[Java] Fix TestSecurity tests on Mac

TestSecurity tests were failing because
InetAddress.getCanonicalHostName was returning
localhost for the 127.0.0.1 IP address which results
Kerberos rejecting as not kinit’d.

Other tests don’t have this issue because they extend
BaseKuduTest which installs the FakeDNS workaround
for 127.* IP addresses.

This patch adds the FakeDNS setup to the TestSecurity
class.

Change-Id: Ic61680f37c2d589ae4f5b553e69bc0b9eb15588d
Reviewed-on: http://gerrit.cloudera.org:8080/10299
Reviewed-by: Dan Burkert 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fb4bc2f6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fb4bc2f6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fb4bc2f6

Branch: refs/heads/branch-1.7.x
Commit: fb4bc2f6422b122b23ca0329c87378546b5f4556
Parents: ffc01ab
Author: Grant Henke 
Authored: Thu May 3 15:42:26 2018 -0500
Committer: Grant Henke 
Committed: Thu May 3 22:04:05 2018 +

--
 .../src/test/java/org/apache/kudu/client/TestSecurity.java | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fb4bc2f6/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
index 762d12c..833d649 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
@@ -36,6 +36,7 @@ import org.apache.kudu.util.SecurityUtil;
 import org.hamcrest.CoreMatchers;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.base.Preconditions;
@@ -83,6 +84,11 @@ public class TestSecurity {
 client.listTabletServers();
   }
 
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+FakeDNS.getInstance().install();
+  }
+
   @After
   public void reset() throws IOException, InterruptedException {
 if (client != null) {

[1/2] kudu git commit: [Java] Fix TestSecurity tests on Mac

2018-05-03 Thread danburkert 
Repository: kudu
Updated Branches:
  refs/heads/master 06afa8f04 -> eb4d88f0a


[Java] Fix TestSecurity tests on Mac

TestSecurity tests were failing because
InetAddress.getCanonicalHostName was returning
localhost for the 127.0.0.1 IP address which results
Kerberos rejecting as not kinit’d.

Other tests don’t have this issue because they extend
BaseKuduTest which installs the FakeDNS workaround
for 127.* IP addresses.

This patch adds the FakeDNS setup to the TestSecurity
class.

Change-Id: Ic61680f37c2d589ae4f5b553e69bc0b9eb15588d
Reviewed-on: http://gerrit.cloudera.org:8080/10298
Reviewed-by: Dan Burkert 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ff4b67e3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ff4b67e3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ff4b67e3

Branch: refs/heads/master
Commit: ff4b67e3b72f08d0e77d1803027ebaad965f6403
Parents: 06afa8f
Author: Grant Henke 
Authored: Thu May 3 15:42:26 2018 -0500
Committer: Grant Henke 
Committed: Thu May 3 21:40:58 2018 +

--
 .../src/test/java/org/apache/kudu/client/TestSecurity.java | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ff4b67e3/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
index 76e59cc..fc851e2 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
@@ -36,6 +36,7 @@ import org.apache.kudu.util.SecurityUtil;
 import org.hamcrest.CoreMatchers;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.base.Preconditions;
@@ -83,6 +84,11 @@ public class TestSecurity {
 client.listTabletServers();
   }
 
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+FakeDNS.getInstance().install();
+  }
+
   @After
   public void reset() throws IOException, InterruptedException {
 if (client != null) {

[2/2] kudu git commit: mini-cluster: support parallel multi-master clusters

2018-05-03 Thread danburkert 
mini-cluster: support parallel multi-master clusters

This commit refactors the mini-clusters to internally use reserved
sockets for their child daemons. Reserved sockets are simply sockets
bound to a random port with SO_REUSEPORT. As a result, master addresses
for multi-master mini clusters no longer need to hard-coded master
ports, and the associated ctest resource lock. This also significantly
lessens the chances that port conflicts will occur, although it is
still possible when masters are restarted during tests. To mitigate
that on Linux, tests are still run in a unique subnet.

Change-Id: I0b0ff7bfc179d8fdb1ed306d1bbd12acddeb060c
Reviewed-on: http://gerrit.cloudera.org:8080/8280
Reviewed-by: Adar Dembo 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eb4d88f0
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eb4d88f0
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eb4d88f0

Branch: refs/heads/master
Commit: eb4d88f0a376b13294c8985d76b3114eaa2e84ca
Parents: ff4b67e
Author: Dan Burkert 
Authored: Sun Oct 15 14:40:46 2017 -0700
Committer: Dan Burkert 
Committed: Thu May 3 21:43:01 2018 +

--
 src/kudu/client/client-test.cc  |   4 +-
 src/kudu/integration-tests/CMakeLists.txt   |  25 ++--
 .../catalog_manager_tsk-itest.cc|   1 -
 .../client-negotiation-failover-itest.cc|   2 -
 .../integration-tests/client-stress-test.cc |   1 -
 .../integration-tests/master-stress-test.cc |   3 +-
 .../master_cert_authority-itest.cc  |  16 +--
 .../integration-tests/master_failover-itest.cc  |   4 +-
 .../integration-tests/master_migration-itest.cc | 107 ---
 .../master_replication-itest.cc |  16 +--
 .../integration-tests/security-faults-itest.cc  |   1 -
 src/kudu/integration-tests/security-itest.cc|   1 -
 .../security-master-auth-itest.cc   |   4 +-
 .../integration-tests/token_signer-itest.cc |  23 ++--
 .../integration-tests/webserver-stress-itest.cc |   9 +-
 src/kudu/master/mini_master.cc  |   1 +
 src/kudu/mini-cluster/CMakeLists.txt|   2 +-
 .../mini-cluster/external_mini_cluster-test.cc  |   6 +-
 src/kudu/mini-cluster/external_mini_cluster.cc  | 119 ++---
 src/kudu/mini-cluster/external_mini_cluster.h   |  20 +--
 src/kudu/mini-cluster/internal_mini_cluster.cc  | 130 +--
 src/kudu/mini-cluster/internal_mini_cluster.h   |  26 +---
 src/kudu/mini-cluster/mini_cluster.cc   |  21 +++
 src/kudu/mini-cluster/mini_cluster.h|  19 ++-
 src/kudu/tools/CMakeLists.txt   |   2 +-
 src/kudu/tools/ksck_remote-test.cc  |   7 +-
 src/kudu/tools/tool_action_test.cc  |   3 -
 27 files changed, 248 insertions(+), 325 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/eb4d88f0/src/kudu/client/client-test.cc
--
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index 816fb39..28f12e2 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -774,7 +774,7 @@ TEST_F(ClientTest, TestBadTable) {
 // Test that, if the master is down, we experience a network error talking
 // to it (no "find the new leader master" since there's only one master).
 TEST_F(ClientTest, TestMasterDown) {
-  cluster_->mini_master()->Shutdown();
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
   shared_ptr t;
   client_->data_->default_admin_operation_timeout_ = MonoDelta::FromSeconds(1);
   Status s = client_->OpenTable("other-tablet", );
@@ -2643,7 +2643,7 @@ void 
ClientTest::DoTestWriteWithDeadServer(WhichServerToKill which) {
   // Shut down the server.
   switch (which) {
 case DEAD_MASTER:
-  cluster_->mini_master()->Shutdown();
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
   break;
 case DEAD_TSERVER:
   cluster_->mini_tablet_server(0)->Shutdown();

http://git-wip-us.apache.org/repos/asf/kudu/blob/eb4d88f0/src/kudu/integration-tests/CMakeLists.txt
--
diff --git a/src/kudu/integration-tests/CMakeLists.txt 
b/src/kudu/integration-tests/CMakeLists.txt
index f5497e2..60ff72a 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -53,7 +53,6 @@ add_dependencies(itest_util
 # Tests
 set(KUDU_TEST_LINK_LIBS itest_util ${KUDU_MIN_TEST_LIBS})
 ADD_KUDU_TEST(all_types-itest
-  RESOURCE_LOCK "master-rpc-ports"
   PROCESSORS 4
   NUM_SHARDS 8)
 ADD_KUDU_TEST(alter_table-randomized-test
@@ -62,10

kudu git commit: docs: workflow for backing up or restoring an entire node

2018-05-03 Thread adar 
Repository: kudu
Updated Branches:
  refs/heads/master f54dbb891 -> 06afa8f04


docs: workflow for backing up or restoring an entire node

I explicitly chose to be vague and not provide example shell commands
because this is such a niche workflow.

Change-Id: I638f4b169282d6688dec414eabfe4c81d0a3f5df
Reviewed-on: http://gerrit.cloudera.org:8080/10223
Reviewed-by: Alexey Serbin 
Reviewed-by: Will Berkeley 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/06afa8f0
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/06afa8f0
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/06afa8f0

Branch: refs/heads/master
Commit: 06afa8f04a8df5563935344a8127ff6a2cd60f8c
Parents: f54dbb8
Author: Adar Dembo 
Authored: Thu Apr 26 15:48:18 2018 -0700
Committer: Adar Dembo 
Committed: Thu May 3 21:39:28 2018 +

--
 docs/administration.adoc | 29 +
 1 file changed, 29 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/06afa8f0/docs/administration.adoc
--
diff --git a/docs/administration.adoc b/docs/administration.adoc
index dd175b4..4c28669 100644
--- a/docs/administration.adoc
+++ b/docs/administration.adoc
@@ -974,3 +974,32 @@ $ rm -rf /data/0/kudu-tserver-wal/* 
/data/0/kudu-tserver-meta/* /data/1/kudu-tse
 . After directories are deleted, the server process can be started with the new
   directory configuration. The appropriate sub-directories will be created by
   Kudu upon starting up.
+
+[[physical_backup]]
+=== Physical backups of an entire node
+
+As documented in the 
link:known_issues.html#_replication_and_backup_limitations[Known Issues and 
Limitations],
+Kudu does not yet provide any built-in backup and restore functionality. 
However,
+it is possible to create a physical backup of a Kudu node (either tablet server
+or master) and restore it later.
+
+WARNING: The node to be backed up must be offline during the procedure, or else
+the backed up (or restored) data will be inconsistent.
+
+WARNING: Certain aspects of the Kudu node (such as its hostname) are embedded 
in
+the on-disk data. As such, it's not yet possible to restore a physical backup 
of
+a node onto another machine.
+
+. Stop all Kudu processes in the cluster. This prevents the tablets on the
+  backed up node from being rereplicated elsewhere unnecessarily.
+
+. If creating a backup, make a copy of the WAL, metadata, and data directories
+  on each node to be backed up. It is important that this copy preserve all 
file
+  attributes as well as sparseness.
+
+. If restoring from a backup, delete the existing WAL, metadata, and data
+  directories, then restore the backup via move or copy. As with creating a
+  backup, it is important that the restore preserve all file attributes and
+  sparseness.
+
+. Start all Kudu processes in the cluster.

kudu git commit: KUDU-2191 (10/n): PoC: disallow Kudu alter column, create table operations from Hive

2018-05-03 Thread danburkert 
Repository: kudu
Updated Branches:
  refs/heads/master 7767679a6 -> f54dbb891


KUDU-2191 (10/n): PoC: disallow Kudu alter column, create table operations from 
Hive

This commit makes it more difficult for processes or users who are not
the Kudu Master to alter the columns of Kudu table HMS entries through
Hive APIs.

The bulk of this patch is introducing a new environment context key/pair
('kudu.master_event=true') which is sent by the Kudu Master with every
HMS create/drop/alter table call. The KuduMetastorePlugin checks that
this property is set appropriately when operations modify a Kudu table
HMS entry.

The result is any attempt to create a table through Hive APIs or alter
the columns of a Kudu table through Hive APIs will fail. These
operations can not be supported because Hive lacks the DDL syntax and
HMS APIs to support necessary Kudu-specific table and column options.

It's possible for a user to spoof this environment context entry, but
it's a big enough roadblock that it should prevent accidental DDL
operations.

Change-Id: Idc39b1df07952d37e9ce93f1673aad535767a8ac
Reviewed-on: http://gerrit.cloudera.org:8080/10111
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f54dbb89
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f54dbb89
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f54dbb89

Branch: refs/heads/master
Commit: f54dbb89183db5559f7dcf76e86239e127f7f023
Parents: 7767679
Author: Dan Burkert 
Authored: Wed Apr 18 17:18:53 2018 -0700
Committer: Dan Burkert 
Committed: Thu May 3 20:31:04 2018 +

--
 .../hive/metastore/KuduMetastorePlugin.java | 34 +++
 .../hive/metastore/TestKuduMetastorePlugin.java | 44 
 src/kudu/hms/hms_catalog.cc | 19 ++---
 src/kudu/hms/hms_catalog.h  |  4 ++
 src/kudu/hms/hms_client-test.cc |  7 +++-
 src/kudu/hms/hms_client.cc  | 18 
 src/kudu/hms/hms_client.h   | 16 ---
 src/kudu/integration-tests/master_hms-itest.cc  | 20 +
 8 files changed, 125 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f54dbb89/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
--
diff --git 
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
 
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
index 4991f10..7a0a8f2 100644
--- 
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
+++ 
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
@@ -17,6 +17,8 @@
 
 package org.apache.kudu.hive.metastore;
 
+import java.util.Map;
+
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
@@ -27,6 +29,7 @@ import 
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
 import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
 import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
 
 /**
  * The {@code KuduMetastorePlugin} intercepts DDL operations on Kudu table 
entries
@@ -61,6 +64,8 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
   static final String KUDU_TABLE_ID_KEY = "kudu.table_id";
   @VisibleForTesting
   static final String KUDU_MASTER_ADDRS_KEY = "kudu.master_addresses";
+  @VisibleForTesting
+  static final String KUDU_MASTER_EVENT = "kudu.master_event";
 
   public KuduMetastorePlugin(Configuration config) {
 super(config);
@@ -78,6 +83,10 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
   return;
 }
 
+if (!isKuduMasterAction(tableEvent)) {
+  throw new MetaException("Kudu tables may not be created through Hive");
+}
+
 checkKuduProperties(table);
   }
 
@@ -94,6 +103,9 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
   return;
 }
 
+// The kudu.master_event property isn't checked, because the kudu.table_id
+// property already implies this event is coming from a Kudu Master.
+
 Table table = tableEvent.getTable();
 
 // Check that the table being dropped is a Kudu table.
@@ -130,6 +142,11 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
 if (!newTableId.equals(oldTableId)) {
   throw new MetaException("Kudu table ID does

kudu git commit: KUDU-2214 Update logging to differentiate voting while copying/tombstoned

2018-05-03 Thread mpercy 
Repository: kudu
Updated Branches:
  refs/heads/master d19844f33 -> 7767679a6


KUDU-2214 Update logging to differentiate voting while copying/tombstoned

Change-Id: I07007601d0a86d6161065629ba167121a33635d6
Reviewed-on: http://gerrit.cloudera.org:8080/10169
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7767679a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7767679a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7767679a

Branch: refs/heads/master
Commit: 7767679a6643743e6db672e68d61b3a0eb70bdca
Parents: d19844f
Author: fwang29 
Authored: Wed May 2 11:40:10 2018 -0700
Committer: Mike Percy 
Committed: Thu May 3 20:17:47 2018 +

--
 src/kudu/consensus/consensus-test-util.h|  4 ++-
 src/kudu/consensus/raft_consensus.cc| 15 ++---
 src/kudu/consensus/raft_consensus.h | 12 ++-
 .../consensus/raft_consensus_quorum-test.cc | 33 +++-
 src/kudu/tserver/tablet_service.cc  |  5 ++-
 5 files changed, 53 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7767679a/src/kudu/consensus/consensus-test-util.h
--
diff --git a/src/kudu/consensus/consensus-test-util.h 
b/src/kudu/consensus/consensus-test-util.h
index 36c2354..2d71b5c 100644
--- a/src/kudu/consensus/consensus-test-util.h
+++ b/src/kudu/consensus/consensus-test-util.h
@@ -544,7 +544,9 @@ class LocalTestPeerProxy : public TestPeerProxy {
 Status s = peers_->GetPeerByUuid(peer_uuid_, );
 
 if (s.ok()) {
-  s = peer->RequestVote(_peer_req, boost::none, _peer_resp);
+  s = peer->RequestVote(_peer_req,
+TabletVotingState(boost::none, 
tablet::TABLET_DATA_READY),
+_peer_resp);
 }
 if (!s.ok()) {
   LOG(WARNING) << "Could not RequestVote from replica with request: "

http://git-wip-us.apache.org/repos/asf/kudu/blob/7767679a/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index f60714a..c8569ac 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -1501,7 +1501,7 @@ void 
RaftConsensus::FillConsensusResponseError(ConsensusResponsePB* response,
 }
 
 Status RaftConsensus::RequestVote(const VoteRequestPB* request,
-  optional tombstone_last_logged_opid,
+  TabletVotingState tablet_voting_state,
   VoteResponsePB* response) {
   TRACE_EVENT2("consensus", "RaftConsensus::RequestVote",
"peer", peer_uuid(),
@@ -1551,15 +1551,20 @@ Status RaftConsensus::RequestVote(const VoteRequestPB* 
request,
   local_last_logged_opid = queue_->GetLastOpIdInLog();
   break;
 default:
-  if (!tombstone_last_logged_opid) {
+  if (!tablet_voting_state.tombstone_last_logged_opid_) {
 return Status::IllegalState("must be running to vote when last-logged 
opid is not known");
   }
   if (!FLAGS_raft_enable_tombstoned_voting) {
 return Status::IllegalState("must be running to vote when tombstoned 
voting is disabled");
   }
-  local_last_logged_opid = *tombstone_last_logged_opid;
-  LOG_WITH_PREFIX_UNLOCKED(INFO) << "voting while tombstoned based on 
last-logged opid "
- << local_last_logged_opid;
+  local_last_logged_opid = 
*(tablet_voting_state.tombstone_last_logged_opid_);
+  if (tablet_voting_state.data_state_ == tablet::TABLET_DATA_COPYING) {
+LOG_WITH_PREFIX_UNLOCKED(INFO) << "voting while copying based on 
last-logged opid "
+   << local_last_logged_opid;
+  } else if (tablet_voting_state.data_state_ == 
tablet::TABLET_DATA_TOMBSTONED) {
+LOG_WITH_PREFIX_UNLOCKED(INFO) << "voting while tombstoned based on 
last-logged opid "
+   << local_last_logged_opid;
+  }
   break;
   }
   DCHECK(local_last_logged_opid.IsInitialized());

http://git-wip-us.apache.org/repos/asf/kudu/blob/7767679a/src/kudu/consensus/raft_consensus.h
--
diff --git a/src/kudu/consensus/raft_consensus.h 
b/src/kudu/consensus/raft_consensus.h
index 6db6774..a92ebc1 100644
--- a/src/kudu/consensus/raft_consensus.h
+++ b/src/kudu/consensus/raft_consensus.h
@@ -42,6 +42,7 @@
 #include "kudu/gutil/macros.h"
 #include "kudu/gutil/port.h"

kudu git commit: tool: improve format for dumping a rowset

2018-05-03 Thread todd 
Repository: kudu
Updated Branches:
  refs/heads/master eded05723 -> d19844f33


tool: improve format for dumping a rowset

This changes the output for 'kudu local_replica dump rowset' to be more
human-readable. The output now looks like this:

RowIdxInBlock: 0; Base: (int32 key=0, int32 int_val=0, string 
string_val="HelloWorld"); Undo Mutations: [@1(DELETE)]; Redo Mutations: [];
RowIdxInBlock: 1; Base: (int32 key=1, int32 int_val=10, string 
string_val="HelloWorld"); Undo Mutations: [@2(DELETE)]; Redo Mutations: [];
RowIdxInBlock: 2; Base: (int32 key=2, int32 int_val=20, string 
string_val="HelloWorld"); Undo Mutations: [@3(DELETE)]; Redo Mutations: [];
...

rather than separately dumping each column block. Dumping individual blocks is
still possible by using the cfile dump commands.

Change-Id: I0f1d08e08d2a3d20a87e49bb5338bf0585bd8e40
Reviewed-on: http://gerrit.cloudera.org:8080/3946
Tested-by: Todd Lipcon 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d19844f3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d19844f3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d19844f3

Branch: refs/heads/master
Commit: d19844f33817c511476746b7a7ec07e6826d52a0
Parents: eded057
Author: Todd Lipcon 
Authored: Mon Apr 23 18:03:24 2018 -0700
Committer: Todd Lipcon 
Committed: Thu May 3 18:05:27 2018 +

--
 src/kudu/tools/kudu-tool-test.cc|  10 +-
 src/kudu/tools/tool_action_local_replica.cc | 230 ++-
 2 files changed, 23 insertions(+), 217 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d19844f3/src/kudu/tools/kudu-tool-test.cc
--
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index d436f9e..fcd4e0f 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -1145,11 +1145,11 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
 ASSERT_STR_CONTAINS(stdout, "bloom_block {");
 ASSERT_STR_MATCHES(stdout, "id: .*");
 ASSERT_STR_CONTAINS(stdout, "undo_deltas {");
-ASSERT_STR_MATCHES(stdout, "CFile Header: ");
-ASSERT_STR_MATCHES(stdout, "Delta stats:.*");
-ASSERT_STR_MATCHES(stdout, "ts range=.*");
-ASSERT_STR_MATCHES(stdout, "update_counts_by_col_id=.*");
-ASSERT_STR_MATCHES(stdout, "Dumping column block.*for column id.*");
+
+ASSERT_STR_CONTAINS(stdout,
+   "RowIdxInBlock: 0; Base: (int32 key=0, int32 int_val=0,"
+   " string string_val=\"HelloWorld\"); "
+   "Undo Mutations: [@1(DELETE)]; Redo Mutations: [];");
 ASSERT_STR_MATCHES(stdout, ".*-.*");
 
 // This is expected to fail with Invalid argument for kRowId.

http://git-wip-us.apache.org/repos/asf/kudu/blob/d19844f3/src/kudu/tools/tool_action_local_replica.cc
--
diff --git a/src/kudu/tools/tool_action_local_replica.cc 
b/src/kudu/tools/tool_action_local_replica.cc
index ce6a951..c65e51e 100644
--- a/src/kudu/tools/tool_action_local_replica.cc
+++ b/src/kudu/tools/tool_action_local_replica.cc
@@ -15,13 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "kudu/tools/tool_action.h"
-
-#include 
 #include 
 #include 
 #include 
-#include 
 #include 
 #include 
 #include 
@@ -34,19 +30,15 @@
 #include 
 #include 
 
-#include "kudu/cfile/cfile.pb.h"
-#include "kudu/cfile/cfile_reader.h"
-#include "kudu/cfile/cfile_util.h"
 #include "kudu/common/common.pb.h"
 #include "kudu/common/partition.h"
-#include "kudu/common/row_changelist.h"
-#include "kudu/common/rowblock.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/wire_protocol.h"
 #include "kudu/consensus/consensus.pb.h"
 #include "kudu/consensus/consensus_meta.h"
 #include "kudu/consensus/consensus_meta_manager.h"
 #include "kudu/consensus/log.pb.h"
+#include "kudu/consensus/log_anchor_registry.h"
 #include "kudu/consensus/log_index.h"
 #include "kudu/consensus/log_reader.h"
 #include "kudu/consensus/log_util.h"
@@ -55,35 +47,28 @@
 #include "kudu/fs/block_id.h"
 #include "kudu/fs/block_manager.h"
 #include "kudu/fs/fs_manager.h"
-#include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/strings/human_readable.h"
 #include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/numbers.h"
-#include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/stringpiece.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/gutil/strings/util.h"
 #include "kudu/master/sys_catalog.h"
 #include

kudu git commit: rpc: add experimental rpc_reuseport flag

2018-05-03 Thread danburkert 
Repository: kudu
Updated Branches:
  refs/heads/master 96fa80503 -> eded05723


rpc: add experimental rpc_reuseport flag

This option sets the SO_REUSEPORT socket option on a server's bound
socket. This socket option is supported on all platforms which Kudu
supports, except RHEL 6.4 and below.

The motivation is to use the option in the minicluster so that master
ports can be reserved by the mini cluster control processes and reused
by the master processes.

When rpc_reuseport is set on RHEL 6.4 the server fails to startup with
the following message (which is expected):

$ cat /etc/redhat-release
CentOS release 6.4 (Final)
$ ./kudu-master --fs-wal-dir=/tmp/kudu-master --logtostderr 
--unlock-experimental-flags --rpc-reuseport=true
...
F0501 14:56:30.672461  3271 master_main.cc:74] Check failed: _s.ok() Bad 
status: Network error: failed to set SO_REUSEPORT: Protocol not available 
(error 92)

Change-Id: I5d8ce9faa646fa2be554f5cfdf8b6ed0c48b496e
Reviewed-on: http://gerrit.cloudera.org:8080/8279
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eded0572
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eded0572
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eded0572

Branch: refs/heads/master
Commit: eded05723e6166b676074c5a548ac1278c7c4401
Parents: 96fa805
Author: Dan Burkert 
Authored: Sun Oct 15 12:16:30 2017 -0700
Committer: Dan Burkert 
Committed: Thu May 3 16:52:50 2018 +

--
 src/kudu/rpc/messenger.cc|  12 +++-
 src/kudu/rpc/messenger.h |   7 +++
 src/kudu/rpc/rpc-test.cc |   4 +-
 src/kudu/server/rpc_server.cc|   7 ++-
 src/kudu/server/rpc_server.h |   1 +
 src/kudu/server/server_base.cc   |   4 ++
 src/kudu/util/net/socket-test.cc |   2 +-
 src/kudu/util/net/socket.cc  | 114 +++---
 src/kudu/util/net/socket.h   |   7 +++
 9 files changed, 88 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/eded0572/src/kudu/rpc/messenger.cc
--
diff --git a/src/kudu/rpc/messenger.cc b/src/kudu/rpc/messenger.cc
index 67cd502..17ac0c5 100644
--- a/src/kudu/rpc/messenger.cc
+++ b/src/kudu/rpc/messenger.cc
@@ -81,7 +81,8 @@ MessengerBuilder::MessengerBuilder(std::string name)
   rpc_encryption_("optional"),
   rpc_tls_ciphers_(kudu::security::SecurityDefaults::kDefaultTlsCiphers),
   
rpc_tls_min_protocol_(kudu::security::SecurityDefaults::kDefaultTlsMinVersion),
-  enable_inbound_tls_(false) {
+  enable_inbound_tls_(false),
+  reuseport_(false) {
 }
 
 MessengerBuilder& MessengerBuilder::set_connection_keepalive_time(const 
MonoDelta ) {
@@ -178,6 +179,11 @@ MessengerBuilder& MessengerBuilder::enable_inbound_tls() {
   return *this;
 }
 
+MessengerBuilder& MessengerBuilder::set_reuseport() {
+  reuseport_ = true;
+  return *this;
+}
+
 Status MessengerBuilder::Build(shared_ptr *msgr) {
   // Initialize SASL library before we start making requests
   RETURN_NOT_OK(SaslInit(!keytab_file_.empty()));
@@ -311,6 +317,9 @@ Status Messenger::AddAcceptorPool(const Sockaddr 
_addr,
   Socket sock;
   RETURN_NOT_OK(sock.Init(0));
   RETURN_NOT_OK(sock.SetReuseAddr(true));
+  if (reuseport_) {
+RETURN_NOT_OK(sock.SetReusePort(true));
+  }
   RETURN_NOT_OK(sock.Bind(accept_addr));
   Sockaddr remote;
   RETURN_NOT_OK(sock.GetSocketAddress());
@@ -402,6 +411,7 @@ Messenger::Messenger(const MessengerBuilder )
 rpc_negotiation_timeout_ms_(bld.rpc_negotiation_timeout_ms_),
 sasl_proto_name_(bld.sasl_proto_name_),
 keytab_file_(bld.keytab_file_),
+reuseport_(bld.reuseport_),
 retain_self_(this) {
   for (int i = 0; i < bld.num_reactors_; i++) {
 reactors_.push_back(new Reactor(retain_self_, i, bld));

http://git-wip-us.apache.org/repos/asf/kudu/blob/eded0572/src/kudu/rpc/messenger.h
--
diff --git a/src/kudu/rpc/messenger.h b/src/kudu/rpc/messenger.h
index 3835cea..64a804b 100644
--- a/src/kudu/rpc/messenger.h
+++ b/src/kudu/rpc/messenger.h
@@ -163,6 +163,9 @@ class MessengerBuilder {
   // Configure the messenger to enable TLS encryption on inbound connections.
   MessengerBuilder& enable_inbound_tls();
 
+  // Configure the messenger to set the SO_REUSEPORT socket option.
+  MessengerBuilder& set_reuseport();
+
   Status Build(std::shared_ptr *msgr);
 
  private:
@@ -185,6 +188,7 @@ class MessengerBuilder {
   std::string rpc_private_key_password_cmd_;
   std::string keytab_file_;
   bool enable_inbound_tls_;
+  bool reuseport_;
 };
 
 // A Messenger is a container for the reactor threads which run event loops
@@

kudu git commit: java: fix error-prone DefaultCharset[1] issues

2018-05-03 Thread granthenke 
Repository: kudu
Updated Branches:
  refs/heads/master 610b1aefb -> 96fa80503


java: fix error-prone DefaultCharset[1] issues

This should be a non-functional change on all systems where UTF8 is the
default character set. It may fix issues on other systems which are
exceedingly rare these days.

[1] 
https://github.com/google/error-prone/blob/master/docs/bugpattern/DefaultCharset.md

Change-Id: Ic92985b5d9466c8629475f3645c872914e3f73a1
Reviewed-on: http://gerrit.cloudera.org:8080/10197
Reviewed-by: Grant Henke 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/96fa8050
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/96fa8050
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/96fa8050

Branch: refs/heads/master
Commit: 96fa805030f859fd0d8d25243191534222163eb5
Parents: 610b1ae
Author: Todd Lipcon 
Authored: Wed Apr 25 11:43:20 2018 -0700
Committer: Grant Henke 
Committed: Thu May 3 16:49:20 2018 +

--
 .../org/apache/kudu/mapreduce/tools/ITImportCsv.java  |  3 +--
 .../java/org/apache/kudu/client/AsyncKuduScanner.java |  3 ++-
 .../org/apache/kudu/client/ColumnRangePredicate.java  |  6 --
 .../java/org/apache/kudu/client/ProtobufHelper.java   |  7 ---
 .../java/org/apache/kudu/client/RemoteTablet.java |  4 +++-
 .../java/org/apache/kudu/client/MiniKuduCluster.java  |  5 -
 .../org/apache/kudu/client/TestAsyncKuduClient.java   |  6 +++---
 .../java/org/apache/kudu/client/TestKeyEncoding.java  | 14 +++---
 .../java/org/apache/kudu/client/TestKuduClient.java   |  3 ++-
 .../org/apache/kudu/client/TestKuduPredicate.java | 12 +---
 .../java/org/apache/kudu/client/TestRowResult.java| 13 +++--
 .../java/org/apache/kudu/client/TestTestUtils.java|  3 ++-
 .../flume/sink/AvroKuduOperationsProducerTest.java|  4 ++--
 .../flume/sink/KeyedKuduOperationsProducerTest.java   |  9 +
 .../java/org/apache/kudu/flume/sink/KuduSinkTest.java |  7 ---
 .../flume/sink/RegexpKuduOperationsProducerTest.java  |  7 ---
 .../org/apache/kudu/mapreduce/ITOutputFormatJob.java  |  2 +-
 .../java/org/apache/kudu/mapreduce/TestJarFinder.java |  7 ---
 18 files changed, 68 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/96fa8050/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
--
diff --git 
a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
 
b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
index 43c0f37..45d50d6 100644
--- 
a/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
+++ 
b/java/kudu-client-tools/src/test/java/org/apache/kudu/mapreduce/tools/ITImportCsv.java
@@ -17,6 +17,7 @@
 
 package org.apache.kudu.mapreduce.tools;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -42,8 +43,6 @@ import org.apache.kudu.client.CreateTableOptions;
 import org.apache.kudu.mapreduce.CommandLineParser;
 import org.apache.kudu.mapreduce.HadoopTestingUtility;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 public class ITImportCsv extends BaseKuduTest {
 
   private static final String TABLE_NAME =

http://git-wip-us.apache.org/repos/asf/kudu/blob/96fa8050/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
--
diff --git 
a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java 
b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
index ceae708..0d13fa6 100644
--- 
a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
+++ 
b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
@@ -27,6 +27,7 @@
 package org.apache.kudu.client;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.kudu.tserver.Tserver.NewScanRequestPB;
 import static org.apache.kudu.tserver.Tserver.ScanRequestPB;
 import static org.apache.kudu.tserver.Tserver.ScanResponsePB;
@@ -620,7 +621,7 @@ public final class AsyncKuduScanner {
   tablet);
 }
 invalidate();
-scannerId = "client debug closed".getBytes();   // Make debugging 
easier.
+scannerId = "client debug closed".getBytes(UTF_8);   // Make debugging 
easier.
 return response == null ? null : response.data;
   }

kudu git commit: [jepsen] make kudu-master-see-tservers more robust

2018-05-03 Thread alexey 
Repository: kudu
Updated Branches:
  refs/heads/master e45fbd1b0 -> 610b1aefb


[jepsen] make kudu-master-see-tservers more robust

Updated kudu-master-see-tservers function to be less fragile.
The prior implementation depended on the format of the output of the
'kudu cluster ksck' sub-command.  The updated implementation uses
'kudu tserver list' and relies only on the number of output lines
by the latter sub-command.

Also, dropped redundant 'node' parameter of the kudu-cli function.

Change-Id: I8b9ff0b7c145e5235a94de27bbedea1fe36f5e60
Reviewed-on: http://gerrit.cloudera.org:8080/10290
Reviewed-by: Will Berkeley 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/610b1aef
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/610b1aef
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/610b1aef

Branch: refs/heads/master
Commit: 610b1aefbce3fc8f727043f31b49d618cc86be03
Parents: e45fbd1
Author: Alexey Serbin 
Authored: Wed May 2 22:18:40 2018 -0700
Committer: Alexey Serbin 
Committed: Thu May 3 16:11:50 2018 +

--
 .../src/main/clojure/jepsen/kudu/util.clj   | 20 +++-
 1 file changed, 11 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/610b1aef/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
--
diff --git a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj 
b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
index ad41b8f..a2b8221 100644
--- a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
+++ b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
@@ -81,7 +81,7 @@
 (defn kudu-cli
   "Returns path to the Kudu CLI tool or just binary name, if it's appropriate
   to rely on the PATH environment variable."
-  [test node]
+  [test]
   (if (:use-packages? test)
 "kudu"  ;; relying on the standard PATH env variable
 (path kudu-target-bin-dir "kudu")))
@@ -124,7 +124,7 @@
   This function should be called in the context of already established SSH
   session at the node."
   [test node]
-  (try (c/exec :sudo :-u kudu-uname (kudu-cli test node) :table :list node)
+  (try (c/exec :sudo :-u kudu-uname (kudu-cli test) :table :list node)
true
(catch RuntimeException _ false)))
 
@@ -132,20 +132,22 @@
 (defn kudu-master-see-tservers?
   "Whether the Kudu master sees the specified number of tablet servers."
   [test node tservers-count]
-  (let [pattern (str "Fetched info from all "
- (str tservers-count)" tablet servers")]
-(try (c/exec :sudo :-u kudu-uname (kudu-cli test node) :cluster :ksck node 
|
- :grep pattern)
+  (let [count-regex (str "^"(str tservers-count)"$")]
+(try (c/exec :sudo :-u kudu-uname
+ (kudu-cli test) :tserver :list node :--format=space |
+ :wc :-l |
+ :tr :-d (c/lit "[:space:]") |
+ :grep :-E count-regex)
  true
  (catch RuntimeException _ false
 
 
 (defn kudu-tserver-in-service?
-  "Is the Kudu tserver process at the specified node is up and running?
+  "Is the Kudu tserver process at the specified node up and running?
   This function should be called in the context of already established SSH
   session at the node."
   [test node]
-  (try (c/exec :sudo :-u kudu-uname (kudu-cli test node) :tserver :status node)
+  (try (c/exec :sudo :-u kudu-uname (kudu-cli test) :tserver :status node)
true
(catch RuntimeException _ false)))
 
@@ -183,7 +185,7 @@
 
   ;; Wait until the master sees all tservers in the cluster. Otherwise
   ;; the client would not be able to create a table with the desired
-  ;; replication factor when not all tservers have registered.
+  ;; replication factor when not all tservers have registered yet.
   (let [tservers-count (count (:tservers test))]
 (loop [iteration 0]
   (when-not (kudu-master-see-tservers? test node tservers-count)