[1/2] kudu git commit: Add ExtractDouble method to JsonReader and small improvements

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 785490ce5 -> b3486d99a


Add ExtractDouble method to JsonReader and small improvements

Change-Id: Iccbdc2467746b68ac871f5feeb01e610610b13d6
Reviewed-on: http://gerrit.cloudera.org:8080/11728
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0a7940d6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0a7940d6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0a7940d6

Branch: refs/heads/master
Commit: 0a7940d687980d1451cd70f7a4b5b80e62797808
Parents: 785490c
Author: Will Berkeley 
Authored: Thu Oct 18 14:24:35 2018 -0700
Committer: Will Berkeley 
Committed: Fri Oct 19 18:34:46 2018 +

--
 src/kudu/tools/kudu-admin-test.cc |  2 +-
 src/kudu/util/jsonreader-test.cc  | 42 
 src/kudu/util/jsonreader.cc   | 72 ++
 src/kudu/util/jsonreader.h|  4 ++
 4 files changed, 103 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/0a7940d6/src/kudu/tools/kudu-admin-test.cc
--
diff --git a/src/kudu/tools/kudu-admin-test.cc 
b/src/kudu/tools/kudu-admin-test.cc
index 22612ac..78adeeb 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -1976,7 +1976,7 @@ TEST_F(AdminCliTest, TestLocateRowMore) {
   // Test providing valid JSON that's not an array.
   NO_FATALS(check_bad_input(
   "{ \"key_hash\" : \"foo\", \"key_range\" : 2 }",
-  "Wrong type during field extraction: expected object array"));
+  "wrong type during field extraction: expected object array"));
 }
 
 TEST_F(AdminCliTest, TestLocateRowAndCheckRowPresence) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/0a7940d6/src/kudu/util/jsonreader-test.cc
--
diff --git a/src/kudu/util/jsonreader-test.cc b/src/kudu/util/jsonreader-test.cc
index 50e1fc6..250091f 100644
--- a/src/kudu/util/jsonreader-test.cc
+++ b/src/kudu/util/jsonreader-test.cc
@@ -56,6 +56,9 @@ TEST(JsonReaderTest, Empty) {
   ASSERT_TRUE(r.ExtractBool(r.root(), "foo", nullptr).IsNotFound());
   ASSERT_TRUE(r.ExtractInt32(r.root(), "foo", nullptr).IsNotFound());
   ASSERT_TRUE(r.ExtractInt64(r.root(), "foo", nullptr).IsNotFound());
+  ASSERT_TRUE(r.ExtractUint32(r.root(), "foo", nullptr).IsNotFound());
+  ASSERT_TRUE(r.ExtractUint64(r.root(), "foo", nullptr).IsNotFound());
+  ASSERT_TRUE(r.ExtractDouble(r.root(), "foo", nullptr).IsNotFound());
   ASSERT_TRUE(r.ExtractString(r.root(), "foo", nullptr).IsNotFound());
   ASSERT_TRUE(r.ExtractObject(r.root(), "foo", nullptr).IsNotFound());
   ASSERT_TRUE(r.ExtractObjectArray(r.root(), "foo", nullptr).IsNotFound());
@@ -72,6 +75,9 @@ TEST(JsonReaderTest, Basic) {
   ASSERT_TRUE(r.ExtractBool(r.root(), "foo", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractInt32(r.root(), "foo", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractInt64(r.root(), "foo", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractUint32(r.root(), "foo", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractUint64(r.root(), "foo", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractDouble(r.root(), "foo", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractObject(r.root(), "foo", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractObjectArray(r.root(), "foo", 
nullptr).IsInvalidArgument());
 }
@@ -112,17 +118,26 @@ TEST(JsonReaderTest, LessBasic) {
   ASSERT_TRUE(r.ExtractBool(r.root(), "null", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractInt32(r.root(), "null", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractInt64(r.root(), "null", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractUint32(r.root(), "null", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractUint64(r.root(), "null", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractDouble(r.root(), "null", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractObject(r.root(), "null", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractObjectArray(r.root(), "null", 
nullptr).IsInvalidArgument());
 
   ASSERT_TRUE(r.ExtractBool(r.root(), "empty", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractInt32(r.root(), "empty", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractInt64(r.root(), "empty", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractUint32(r.root(), "empty", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractUint64(r.root(), "empty", nullptr).IsInvalidArgument());
+  ASSERT_TRUE(r.ExtractDouble(r.root(), "empty", nullptr).IsInvalidArgument());
   ASSERT_TRUE(r.ExtractObject(r.root(), "empty", nullptr).IsInvalidArgument());
   

[2/2] kudu git commit: Add missing test dependency for Hive tests

[danburkert]

Add missing test dependency for Hive tests

The HMS client internally calls into the JobConf class, which is
provided by the org.apache.hadoop.hadoop-mapreduce-client-core artifact.
The HMS only specifies an optional dependency on
hadoop-mapreduce-client-core, so we are forced to depend on it
explicitly.

Change-Id: I33187880247b3fe930ff521b0671345646e0fbf4
Reviewed-on: http://gerrit.cloudera.org:8080/11733
Reviewed-by: Hao Hao 
Reviewed-by: Andrew Wong 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b3486d99
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b3486d99
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b3486d99

Branch: refs/heads/master
Commit: b3486d99a1e1bb5debef82912728d522f58621b7
Parents: 0a7940d
Author: Dan Burkert 
Authored: Thu Oct 18 17:18:32 2018 -0700
Committer: Dan Burkert 
Committed: Fri Oct 19 19:23:54 2018 +

--
 java/gradle/dependencies.gradle | 1 +
 java/kudu-client/build.gradle   | 4 
 2 files changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/b3486d99/java/gradle/dependencies.gradle
--
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index 267e452..52a5079 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -83,6 +83,7 @@ libs += [
 hadoopClient : "org.apache.hadoop:hadoop-client:$versions.hadoop",
 hadoopCommon : "org.apache.hadoop:hadoop-common:$versions.hadoop",
 hadoopMRClientCommon : 
"org.apache.hadoop:hadoop-mapreduce-client-common:$versions.hadoop",
+hadoopMRClientCore   : 
"org.apache.hadoop:hadoop-mapreduce-client-core:$versions.hadoop",
 hamcrestCore : "org.hamcrest:hamcrest-core:$versions.hamcrest",
 hiveMetastore: "org.apache.hive:hive-metastore:$versions.hive",
 hiveMetastoreTest: 
"org.apache.hive:hive-metastore:$versions.hive:tests",

http://git-wip-us.apache.org/repos/asf/kudu/blob/b3486d99/java/kudu-client/build.gradle
--
diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle
index 363135b..2b99355 100644
--- a/java/kudu-client/build.gradle
+++ b/java/kudu-client/build.gradle
@@ -38,6 +38,10 @@ dependencies {
   testCompile project(":kudu-test-utils")
   testCompile libs.hamcrestCore
   testCompile libs.hiveMetastore
+  // The HMS client relies on the MR client-core artifact for JobConf, but only
+  // specifies it as an optional dependency. Gradle doesn't pull in optional
+  // dependencies, so we have to depend on it directly.
+  testCompile libs.hadoopMRClientCore
   testCompile libs.junit
   testCompile libs.log4j
   testCompile libs.mockitoCore

[2/3] kudu git commit: thrift operators: add more test coverage

[danburkert]

thrift operators: add more test coverage

This is a follow-up to Adar's suggestions in
https://gerrit.cloudera.org/c/11693/.

Change-Id: Ifdba996a9c4e6e01f55c257bfd10fb3f74f0f894
Reviewed-on: http://gerrit.cloudera.org:8080/11713
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1e8ef0ea
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1e8ef0ea
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1e8ef0ea

Branch: refs/heads/master
Commit: 1e8ef0eac6d411a225bd3a4101a7e19346eb1048
Parents: 7607e23
Author: Dan Burkert 
Authored: Wed Oct 17 12:53:05 2018 -0700
Committer: Dan Burkert 
Committed: Wed Oct 17 23:02:28 2018 +

--
 src/kudu/sentry/thrift_operators-test.cc | 56 +--
 1 file changed, 36 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1e8ef0ea/src/kudu/sentry/thrift_operators-test.cc
--
diff --git a/src/kudu/sentry/thrift_operators-test.cc 
b/src/kudu/sentry/thrift_operators-test.cc
index 4ef3873..d4aa070 100644
--- a/src/kudu/sentry/thrift_operators-test.cc
+++ b/src/kudu/sentry/thrift_operators-test.cc
@@ -65,32 +65,48 @@ TEST(ThriftOperatorsTest, TestOperatorLt) {
   ASSERT_EQ(2, groups.size()) << groups;
 
   // TSentryPrivilege::operator<
+  const string kServer = "server1";
+  const string kDatabase = "db1";
+
   TSentryPrivilege db_priv;
-  db_priv.__set_serverName("server1");
-  db_priv.__set_dbName("db1");
+  db_priv.__set_serverName(kServer);
+  db_priv.__set_dbName(kDatabase);
 
-  TSentryPrivilege tbl_priv;
-  tbl_priv.__set_serverName("server1");
-  tbl_priv.__set_dbName("db1");
-  tbl_priv.__set_tableName("tbl1");
+  TSentryPrivilege tbl1_priv;
+  tbl1_priv.__set_serverName(kServer);
+  tbl1_priv.__set_dbName(kDatabase);
+  tbl1_priv.__set_tableName("tbl1");
 
-  NO_FATALS(AssertCompareRequirements(db_priv, tbl_priv));
-  set privileges { db_priv, tbl_priv };
-  ASSERT_EQ(2, privileges.size()) << privileges;
+  TSentryPrivilege tbl2_priv;
+  tbl2_priv.__set_serverName(kServer);
+  tbl2_priv.__set_dbName(kDatabase);
+  tbl2_priv.__set_tableName("tbl2");
 
+  NO_FATALS(AssertCompareRequirements(db_priv, tbl1_priv));
+  NO_FATALS(AssertCompareRequirements(db_priv, tbl2_priv));
+  NO_FATALS(AssertCompareRequirements(tbl1_priv, tbl2_priv));
+  set privileges { db_priv, tbl1_priv, tbl2_priv };
+  ASSERT_EQ(3, privileges.size()) << privileges;
 
   // TSentryAuthorizable::operator<
   TSentryAuthorizable db_authorizable;
-  db_authorizable.__set_server("server1");
-  db_authorizable.__set_db("db1");
-
-  TSentryAuthorizable tbl_authorizable;
-  tbl_authorizable.__set_server("server1");
-  tbl_authorizable.__set_db("db1");
-  tbl_authorizable.__set_table("tbl1");
-
-  NO_FATALS(AssertCompareRequirements(db_authorizable, tbl_authorizable));
-  set authorizables { db_authorizable, tbl_authorizable };
-  ASSERT_EQ(2, authorizables.size()) << authorizables;
+  db_authorizable.__set_server(kServer);
+  db_authorizable.__set_db(kDatabase);
+
+  TSentryAuthorizable tbl1_authorizable;
+  tbl1_authorizable.__set_server(kServer);
+  tbl1_authorizable.__set_db(kDatabase);
+  tbl1_authorizable.__set_table("tbl1");
+
+  TSentryAuthorizable tbl2_authorizable;
+  tbl2_authorizable.__set_server(kServer);
+  tbl2_authorizable.__set_db(kDatabase);
+  tbl2_authorizable.__set_table("tbl2");
+
+  NO_FATALS(AssertCompareRequirements(db_authorizable, tbl1_authorizable));
+  NO_FATALS(AssertCompareRequirements(db_authorizable, tbl2_authorizable));
+  NO_FATALS(AssertCompareRequirements(tbl1_authorizable, tbl2_authorizable));
+  set authorizables { db_authorizable, tbl1_authorizable, 
tbl2_authorizable };
+  ASSERT_EQ(3, authorizables.size()) << authorizables;
 }
 } // namespace sentry

[1/3] kudu git commit: sentry module: introduce using declarations for generated types

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 36d09f187 -> 082bbfe20


sentry module: introduce using declarations for generated types

This cleans up the .cc files in the sentry module with 'using'
declarations, as suggested by Adar.

Change-Id: I51f61eedc1069d362fa1ce26ce0449cce8f0846b
Reviewed-on: http://gerrit.cloudera.org:8080/11712
Reviewed-by: Adar Dembo 
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7607e235
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7607e235
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7607e235

Branch: refs/heads/master
Commit: 7607e2354bf61957e4d3168d75d4990169dc2472
Parents: 36d09f1
Author: Dan Burkert 
Authored: Wed Oct 17 12:46:03 2018 -0700
Committer: Dan Burkert 
Committed: Wed Oct 17 23:02:24 2018 +

--
 src/kudu/sentry/sentry_client-test.cc| 55 ---
 src/kudu/sentry/sentry_client.cc | 55 ---
 src/kudu/sentry/thrift_operators-test.cc | 34 +++--
 3 files changed, 79 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7607e235/src/kudu/sentry/sentry_client-test.cc
--
diff --git a/src/kudu/sentry/sentry_client-test.cc 
b/src/kudu/sentry/sentry_client-test.cc
index d8c845d..10a4134 100644
--- a/src/kudu/sentry/sentry_client-test.cc
+++ b/src/kudu/sentry/sentry_client-test.cc
@@ -35,6 +35,17 @@
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
+using sentry::TAlterSentryRoleAddGroupsRequest;
+using sentry::TAlterSentryRoleAddGroupsResponse;
+using sentry::TAlterSentryRoleGrantPrivilegeRequest;
+using sentry::TAlterSentryRoleGrantPrivilegeResponse;
+using sentry::TCreateSentryRoleRequest;
+using sentry::TDropSentryRoleRequest;
+using sentry::TListSentryPrivilegesRequest;
+using sentry::TListSentryPrivilegesResponse;
+using sentry::TSentryAuthorizable;
+using sentry::TSentryGroup;
+using sentry::TSentryPrivilege;
 using std::set;
 using std::string;
 using std::unique_ptr;
@@ -106,12 +117,12 @@ TEST_P(SentryClientTest, TestMiniSentryLifecycle) {
  sentry_client_opts));
   auto smoketest = [&]() -> Status {
 return client.Execute([](SentryClient* client) -> Status {
-::sentry::TCreateSentryRoleRequest create_req;
+TCreateSentryRoleRequest create_req;
 create_req.requestorUserName = "test-admin";
 create_req.roleName = "test-role";
 RETURN_NOT_OK(client->CreateRole(create_req));
 
-::sentry::TDropSentryRoleRequest drop_req;
+TDropSentryRoleRequest drop_req;
 drop_req.requestorUserName = "test-admin";
 drop_req.roleName = "test-role";
 RETURN_NOT_OK(client->DropRole(drop_req));
@@ -137,7 +148,7 @@ TEST_P(SentryClientTest, TestMiniSentryLifecycle) {
 TEST_P(SentryClientTest, TestCreateDropRole) {
 
   { // Create a role
-::sentry::TCreateSentryRoleRequest req;
+TCreateSentryRoleRequest req;
 req.requestorUserName = "test-admin";
 req.roleName = "viewer";
 ASSERT_OK(sentry_client_->CreateRole(req));
@@ -148,7 +159,7 @@ TEST_P(SentryClientTest, TestCreateDropRole) {
   }
 
   { // Attempt to create a role as a non-admin user.
-::sentry::TCreateSentryRoleRequest req;
+TCreateSentryRoleRequest req;
 req.requestorUserName = "joe-interloper";
 req.roleName = "fuzz";
 Status s = sentry_client_->CreateRole(req);
@@ -156,7 +167,7 @@ TEST_P(SentryClientTest, TestCreateDropRole) {
   }
 
   { // Attempt to drop the role as a non-admin user.
-::sentry::TDropSentryRoleRequest req;
+TDropSentryRoleRequest req;
 req.requestorUserName = "joe-interloper";
 req.roleName = "viewer";
 Status s = sentry_client_->DropRole(req);
@@ -164,7 +175,7 @@ TEST_P(SentryClientTest, TestCreateDropRole) {
   }
 
   { // Drop the role
-::sentry::TDropSentryRoleRequest req;
+TDropSentryRoleRequest req;
 req.requestorUserName = "test-admin";
 req.roleName = "viewer";
 ASSERT_OK(sentry_client_->DropRole(req));
@@ -180,15 +191,15 @@ TEST_P(SentryClientTest, TestCreateDropRole) {
 // instances.
 TEST_P(SentryClientTest, TestListPrivilege) {
   // Attempt to access Sentry privileges by a non admin user.
-  ::sentry::TSentryAuthorizable authorizable;
+  TSentryAuthorizable authorizable;
   authorizable.server = "server";
   authorizable.db = "db";
   authorizable.table = "table";
-  ::sentry::TListSentryPrivilegesRequest request;
+  TListSentryPrivilegesRequest request;
   request.requestorUserName = "joe-interloper";
   request.authorizableHierarchy = authorizable;
   request.__set_principalName("viewer");
-  ::sentry::TListSentryPrivilegesResponse response;
+  

[3/3] kudu git commit: HMS integration: provide Java API to override owner during table create

[danburkert]

HMS integration: provide Java API to override owner during table create

This is necessary so that Impala can override the table owner, instead
of using the inferred username of the connection (which would be
'impala'). This API is marked as unstable since it's not clear we will
want to support it long term, and there are some open questions about
the fine-grained privileges required to use it.

Change-Id: I21069d81a4b54e55c399933b69789bf83bd58de0
Reviewed-on: http://gerrit.cloudera.org:8080/11413
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/082bbfe2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/082bbfe2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/082bbfe2

Branch: refs/heads/master
Commit: 082bbfe2048279518a9103e4edcd91f815705294
Parents: 1e8ef0e
Author: Dan Burkert 
Authored: Mon Sep 10 16:02:54 2018 -0700
Committer: Dan Burkert 
Committed: Wed Oct 17 23:02:37 2018 +

--
 java/kudu-client/build.gradle   |  1 +
 .../apache/kudu/client/CreateTableOptions.java  | 23 +++
 .../client/TestHiveMetastoreIntegration.java| 72 
 src/kudu/master/catalog_manager.cc  |  4 +-
 src/kudu/master/master.proto|  4 ++
 5 files changed, 102 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/082bbfe2/java/kudu-client/build.gradle
--
diff --git a/java/kudu-client/build.gradle b/java/kudu-client/build.gradle
index 077e050..363135b 100644
--- a/java/kudu-client/build.gradle
+++ b/java/kudu-client/build.gradle
@@ -37,6 +37,7 @@ dependencies {
 
   testCompile project(":kudu-test-utils")
   testCompile libs.hamcrestCore
+  testCompile libs.hiveMetastore
   testCompile libs.junit
   testCompile libs.log4j
   testCompile libs.mockitoCore

http://git-wip-us.apache.org/repos/asf/kudu/blob/082bbfe2/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
--
diff --git 
a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java 
b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
index cb2ba52..2f137a8 100644
--- 
a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
+++ 
b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
@@ -19,6 +19,8 @@ package org.apache.kudu.client;
 
 import java.util.List;
 
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -209,6 +211,27 @@ public class CreateTableOptions {
 return this;
   }
 
+  /**
+   * Set the table owner as the provided username in configured external 
catalogs
+   * such as the Hive Metastore. Overrides the default of the currently 
logged-in
+   * username or Kerberos principal.
+   *
+   * This is an unstable method because it is not yet clear whether this should
+   * be supported directly in the long run, rather than requiring the table 
creator
+   * to re-assign ownership explicitly.
+   *
+   * @param owner the username to set as the table owner in external catalogs
+   * @return this instance
+   */
+  @InterfaceAudience.LimitedPrivate("Impala")
+  @InterfaceStability.Unstable
+  public CreateTableOptions setOwner(String owner) {
+Preconditions.checkArgument(!Strings.isNullOrEmpty(owner),
+"table owner must not be null or empty");
+pb.setOwner(owner);
+return this;
+  }
+
   Master.CreateTableRequestPB.Builder getBuilder() {
 if (!splitRows.isEmpty() || !rangePartitions.isEmpty()) {
   pb.setSplitRowsRangeBounds(new Operation.OperationsEncoder()

http://git-wip-us.apache.org/repos/asf/kudu/blob/082bbfe2/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java
 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java
new file mode 100644
index 000..daf2396
--- /dev/null
+++ 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestHiveMetastoreIntegration.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 

kudu git commit: Fix thrift operator< implementations

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 09848055b -> 58ecdd155


Fix thrift operator< implementations

The previous implementation didn't account for unset optional fields,
and did not short-circuit after finding the first field which was
smaller. Buggy operator< is problematic because the std::map and
std::set types require the operator< to maintain a total ordering; if it
doesn't then the set and map invariants will not hold.

Change-Id: I1aacf602c603b05433c357a4a236ba0b9e521392
Reviewed-on: http://gerrit.cloudera.org:8080/11693
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/58ecdd15
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/58ecdd15
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/58ecdd15

Branch: refs/heads/master
Commit: 58ecdd15553178b25612f65de71ef25d07058079
Parents: 0984805
Author: Dan Burkert 
Authored: Mon Oct 15 17:44:27 2018 -0700
Committer: Dan Burkert 
Committed: Wed Oct 17 00:01:56 2018 +

--
 src/kudu/sentry/CMakeLists.txt   |   1 +
 src/kudu/sentry/thrift_operators-test.cc | 102 ++
 src/kudu/sentry/thrift_operators.cc  |  38 ++
 3 files changed, 127 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/58ecdd15/src/kudu/sentry/CMakeLists.txt
--
diff --git a/src/kudu/sentry/CMakeLists.txt b/src/kudu/sentry/CMakeLists.txt
index 3222793..fee3855 100644
--- a/src/kudu/sentry/CMakeLists.txt
+++ b/src/kudu/sentry/CMakeLists.txt
@@ -78,4 +78,5 @@ if (NOT NO_TESTS)
 
   ADD_KUDU_TEST(sentry_action-test)
   ADD_KUDU_TEST(sentry_client-test)
+  ADD_KUDU_TEST(thrift_operators-test)
 endif()

http://git-wip-us.apache.org/repos/asf/kudu/blob/58ecdd15/src/kudu/sentry/thrift_operators-test.cc
--
diff --git a/src/kudu/sentry/thrift_operators-test.cc 
b/src/kudu/sentry/thrift_operators-test.cc
new file mode 100644
index 000..46bd6c8
--- /dev/null
+++ b/src/kudu/sentry/thrift_operators-test.cc
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include "kudu/sentry/sentry_policy_service_types.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+using std::string;
+using std::set;
+
+namespace kudu {
+namespace sentry {
+
+class ThriftOperatorsTest : public KuduTest {
+};
+
+template
+void AssertCompareRequirements(const T& a, const T& b) {
+  // Values must not be less than themselves.
+  ASSERT_FALSE(a < a) << a;
+  ASSERT_FALSE(b < b) << b;
+
+  // Two values may not be simultaneously less than each other.
+  if (a < b) {
+ASSERT_FALSE(b < a);
+  }
+}
+
+TEST_F(ThriftOperatorsTest, TestOperatorLt) {
+  // TSentryRole::operator<
+  ::sentry::TSentryRole role_a;
+  role_a.__set_roleName("a");
+
+  ::sentry::TSentryRole role_b;
+  role_b.__set_roleName("b");
+
+  NO_FATALS(AssertCompareRequirements(role_a, role_b));
+  set<::sentry::TSentryRole> roles { role_a, role_b };
+  ASSERT_EQ(2, roles.size()) << roles;
+
+  // TSentryGroup::operator<
+  ::sentry::TSentryGroup group_a;
+  group_a.__set_groupName("a");
+
+  ::sentry::TSentryGroup group_b;
+  group_b.__set_groupName("b");
+
+  NO_FATALS(AssertCompareRequirements(group_a, group_b));
+  set<::sentry::TSentryGroup> groups { group_a, group_b };
+  ASSERT_EQ(2, groups.size()) << groups;
+
+  // TSentryPrivilege::operator<
+  ::sentry::TSentryPrivilege db_priv;
+  db_priv.__set_serverName("server1");
+  db_priv.__set_dbName("db1");
+
+  ::sentry::TSentryPrivilege tbl_priv;
+  tbl_priv.__set_serverName("server1");
+  tbl_priv.__set_dbName("db1");
+  tbl_priv.__set_tableName("tbl1");
+
+  NO_FATALS(AssertCompareRequirements(db_priv, tbl_priv));
+  set<::sentry::TSentryPrivilege> privileges { db_priv, tbl_priv };
+  ASSERT_EQ(2, privileges.size()) << privileges;
+
+
+  // TSentryAuthorizable::operator<
+  

[1/2] kudu git commit: [compaction] KUDU-2056: Expose a metric for how much a tablet needs to be compacted

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 8c184e2a9 -> 09848055b


[compaction] KUDU-2056: Expose a metric for how much a tablet needs to be 
compacted

This adds a new metric 'average_diskrowset_height' that reflects how
uncompacted a tablet replica is. This metric is obtained by integrating
the height function with respect to the by-data-size probability
distribution used by the compaction policy.

To implement the integration, I piggy-backed on the function that
computes the CDF for the rowset layout, since computing the integral
requires computing the CDF and it seemed wasteful to first compute the
CDF, then go through an almost entirely similar bit of logic to compute
the average height.

Change-Id: I98493b901d37bb278167ba2fe98d322a86a1f0f9
Reviewed-on: http://gerrit.cloudera.org:8080/11639
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d3684a7b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d3684a7b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d3684a7b

Branch: refs/heads/master
Commit: d3684a7b2add8f06b7189adb9ce9222b8ae1eff5
Parents: 8c184e2
Author: Will Berkeley 
Authored: Mon Oct 8 16:09:41 2018 -0700
Committer: Will Berkeley 
Committed: Tue Oct 16 20:47:13 2018 +

--
 src/kudu/tablet/compaction_policy-test.cc | 166 +
 src/kudu/tablet/compaction_policy.cc  |   2 +-
 src/kudu/tablet/rowset_info.cc| 101 +--
 src/kudu/tablet/rowset_info.h |  19 +--
 src/kudu/tablet/tablet.cc |  33 -
 src/kudu/tablet/tablet_metrics.cc |  10 +-
 src/kudu/tablet/tablet_metrics.h  |   9 +-
 7 files changed, 287 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d3684a7b/src/kudu/tablet/compaction_policy-test.cc
--
diff --git a/src/kudu/tablet/compaction_policy-test.cc 
b/src/kudu/tablet/compaction_policy-test.cc
index 4277f9d..8ec0829 100644
--- a/src/kudu/tablet/compaction_policy-test.cc
+++ b/src/kudu/tablet/compaction_policy-test.cc
@@ -21,6 +21,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -33,6 +34,7 @@
 #include "kudu/tablet/compaction_policy.h"
 #include "kudu/tablet/mock-rowsets.h"
 #include "kudu/tablet/rowset.h"
+#include "kudu/tablet/rowset_info.h"
 #include "kudu/tablet/rowset_tree.h"
 #include "kudu/util/env.h"
 #include "kudu/util/faststring.h"
@@ -276,5 +278,169 @@ TEST_F(TestCompactionPolicy, KUDU2251) {
   ASSERT_LT(quality, 2.0);
 }
 
+namespace {
+double ComputeAverageRowsetHeight(
+const vector>& intervals) {
+  RowSetVector rowsets;
+  for (const auto& interval : intervals) {
+rowsets.push_back(std::make_shared(interval.first,
+   interval.second));
+  }
+  RowSetTree tree;
+  CHECK_OK(tree.Reset(rowsets));
+
+  double avg_height;
+  RowSetInfo::ComputeCdfAndCollectOrdered(tree, _height, nullptr, nullptr);
+  return avg_height;
+}
+} // anonymous namespace
+
+class KeySpaceCdfTest : public KuduTest {
+ protected:
+  static void AssertWithinEpsilon(double epsilon,
+  double expected,
+  double actual) {
+ASSERT_GE(actual, expected - epsilon);
+ASSERT_LE(actual, expected + epsilon);
+  }
+};
+
+// Test the computation of average rowset heights. This isn't strictly used in
+// compaction policy, but this is a convenient place to put it, for now.
+TEST_F(KeySpaceCdfTest, TestComputingAverageRowSetHeight) {
+  // No rowsets.
+  EXPECT_EQ(0.0, ComputeAverageRowsetHeight({ }));
+
+  /* A rowset that's one key wide.
+   * |
+   */
+  EXPECT_EQ(0.0, ComputeAverageRowsetHeight({ { "A", "A" } }));
+
+  /* A single rowset.
+   * [ --- ]
+   */
+  EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" } }));
+
+  /* Two rowsets with no empty space between.
+   * [ --- ][ --- ]
+   */
+  EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" }, { "B", "C" } }));
+
+
+  /* Three rowsets with no empty spaces between.
+   * [ --- ][ --- ][ --- ]
+   */
+  EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" },
+  { "B", "C" },
+  { "C", "D" } }));
+
+  /* Two rowsets with empty space between them.
+   * [ --- ]   [ --- ]
+   */
+  EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" }, { "C", "D" } }));
+
+  /* Three rowsets with empty space between them.
+   * [ --- ]   [ --- ]   [ --- ]
+   */
+  EXPECT_EQ(1.0, ComputeAverageRowsetHeight({ { "A", "B" },
+  { "C", "D" },
+  { "E", 

[2/2] kudu git commit: Update Java README Intellij test runner instructions

[danburkert]

Update Java README Intellij test runner instructions

Change-Id: Ie372417350f50a7d4f5567cd991ac5d886329cdb
Reviewed-on: http://gerrit.cloudera.org:8080/11700
Reviewed-by: Grant Henke 
Tested-by: Grant Henke 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/09848055
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/09848055
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/09848055

Branch: refs/heads/master
Commit: 09848055b089be98754b82f6cae74ddc78f6
Parents: d3684a7
Author: Dan Burkert 
Authored: Tue Oct 16 13:31:40 2018 -0700
Committer: Dan Burkert 
Committed: Tue Oct 16 20:58:27 2018 +

--
 java/README.adoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/09848055/java/README.adoc
--
diff --git a/java/README.adoc b/java/README.adoc
index 5049b51..caccea8 100644
--- a/java/README.adoc
+++ b/java/README.adoc
@@ -140,7 +140,8 @@ to import the Gradle project.
 
 For the most consistent behavior on the command line and
 in the IDE, be sure to enable `Delegate IDE build/run actions to gradle`
-as described 
https://www.jetbrains.com/help/idea/gradle.html#delegate_build_gradle[here].
+and run tests using the `Gradle Test Runner` as described
+https://www.jetbrains.com/help/idea/gradle.html#delegate_build_gradle[here].
 
 === Eclipse
 

kudu git commit: Implement BloomFilter Predicate in server side.

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master c8dd7b53f -> 8af288a26


Implement BloomFilter Predicate in server side.

Change-Id: I62c2de42667d0255d94e19db773240f7f9ee636c
Reviewed-on: http://gerrit.cloudera.org:8080/11100
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8af288a2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8af288a2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8af288a2

Branch: refs/heads/master
Commit: 8af288a26a204e2acfc3aa4e642fba7de56b43bb
Parents: c8dd7b5
Author: triplesheep 
Authored: Wed Aug 1 10:54:21 2018 +
Committer: Dan Burkert 
Committed: Thu Oct 11 22:50:37 2018 +

--
 src/kudu/common/column_predicate-test.cc | 287 ++
 src/kudu/common/column_predicate.cc  | 193 +++--
 src/kudu/common/column_predicate.h   | 140 -
 src/kudu/common/common.proto |  40 +++-
 src/kudu/common/key_util.cc  |   2 +
 src/kudu/common/scan_spec.cc |   4 +
 src/kudu/common/wire_protocol-test.cc| 128 +++-
 src/kudu/common/wire_protocol.cc |  79 ++-
 src/kudu/tablet/cfile_set-test.cc| 211 ++-
 src/kudu/util/bloom_filter.h |  23 ++-
 10 files changed, 1068 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8af288a2/src/kudu/common/column_predicate-test.cc
--
diff --git a/src/kudu/common/column_predicate-test.cc 
b/src/kudu/common/column_predicate-test.cc
index d96cc41..e0b6b63 100644
--- a/src/kudu/common/column_predicate-test.cc
+++ b/src/kudu/common/column_predicate-test.cc
@@ -30,8 +30,10 @@
 #include "kudu/common/schema.h"
 #include "kudu/common/types.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/bloom_filter.h"
 #include "kudu/util/int128.h"
 #include "kudu/util/memory/arena.h"
+#include "kudu/util/random.h"
 #include "kudu/util/slice.h"
 #include "kudu/util/test_util.h"
 
@@ -41,6 +43,7 @@ namespace kudu {
 
 class TestColumnPredicate : public KuduTest {
  public:
+  TestColumnPredicate() : rand_(SeedRandom()) {}
 
   // Test that when a is merged into b and vice versa, the result is equal to
   // expected, and the resulting type is equal to type.
@@ -68,6 +71,42 @@ class TestColumnPredicate : public KuduTest {
 ASSERT_EQ(b_base.predicate_type(), type);
   }
 
+  void FillBloomFilterAndValues(int n_keys,
+vector* values,
+BloomFilterBuilder* bfb1,
+BloomFilterBuilder* bfb2) {
+uint64_t current = 0;
+for (int i = 0; i < 2; ++i) {
+  while (true) {
+uint64_t key = rand_.Next();
+if (key <= current) {
+  continue;
+}
+current = key;
+Slice key_slice(reinterpret_cast(), sizeof(key));
+BloomKeyProbe probe(key_slice, MURMUR_HASH_2);
+bfb1->AddKey(probe);
+bfb2->AddKey(probe);
+values->emplace_back(key);
+break;
+  }
+}
+for (int i = 2; i < n_keys; ++i) {
+  while (true) {
+uint64_t key = rand_.Next();
+Slice key_slice(reinterpret_cast(), sizeof(key));
+BloomKeyProbe probe(key_slice, MURMUR_HASH_2);
+BloomFilter bf(bfb1->slice(), bfb1->n_hashes());
+if (!bf.MayContainKey(probe) && key > current) {
+  current = key;
+  values->emplace_back(key);
+  bfb2->AddKey(probe);
+  break;
+}
+  }
+}
+  }
+
   template 
   void TestMergeCombinations(const ColumnSchema& column, vector values) {
 // Range + Range
@@ -744,6 +783,184 @@ class TestColumnPredicate : public KuduTest {
   ColumnPredicate::IsNull(column),
   PredicateType::IsNull);
   }
+
+  template 
+  void TestMergeBloomFilterCombinations(const ColumnSchema& column,
+
vector* bf,
+vector values) {
+vector orig_bloom_filters = *bf;
+// BloomFilter AND
+// NONE
+// =
+// NONE
+TestMerge(ColumnPredicate::InBloomFilter(column, bf, nullptr, nullptr),
+  ColumnPredicate::None(column),
+  ColumnPredicate::None(column),
+  PredicateType::None);
+
+// BloomFilter AND
+// Equality
+// =
+// Equality
+*bf = orig_bloom_filters;
+TestMerge(ColumnPredicate::InBloomFilter(column, bf, nullptr, nullptr),
+  ColumnPredicate::Equality(column, [0]),
+  ColumnPredicate::Equality(column, [0]),
+  PredicateType::Equality);
+
+// BloomFilter AND
+// Equality
+// =
+// None
+*bf = 

kudu git commit: Improve logging for mini HMS and Sentry services

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 88edfc36a -> 61aed1a58


Improve logging for mini HMS and Sentry services

This commit adds custom logging configurations for both mini HMS and
Sentry. Both services now log warnings or errors to the console, and
info level messages to a file in the mini cluster root. This cuts down
on verbosity when both are active, and simplifies debugging by
separating full-fidelity logs.

The HMS console log is set to ERROR level and the Sentry console log is
set to the WARN level, based on the relative verbosity of the service
logs. Adjusting these levels as necessary in the future will be
straightforward.

Change-Id: Idd14cda83d56b0588667b9a3de260ffa62842adc
Reviewed-on: http://gerrit.cloudera.org:8080/11635
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao 
Reviewed-by: Andrew Wong 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/61aed1a5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/61aed1a5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/61aed1a5

Branch: refs/heads/master
Commit: 61aed1a58d1510d65cb9a3370ffbe93431e54d58
Parents: 88edfc3
Author: Dan Burkert 
Authored: Mon Oct 8 11:24:14 2018 -0700
Committer: Dan Burkert 
Committed: Wed Oct 10 00:31:17 2018 +

--
 src/kudu/hms/mini_hms.cc   | 38 +
 src/kudu/hms/mini_hms.h|  3 +++
 src/kudu/sentry/mini_sentry.cc | 29 ++--
 3 files changed, 64 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/61aed1a5/src/kudu/hms/mini_hms.cc
--
diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc
index 079cdb1..771cc30 100644
--- a/src/kudu/hms/mini_hms.cc
+++ b/src/kudu/hms/mini_hms.cc
@@ -102,6 +102,7 @@ Status MiniHms::Start() {
 
   RETURN_NOT_OK(CreateHiveSite());
   RETURN_NOT_OK(CreateCoreSite());
+  RETURN_NOT_OK(CreateLogConfig());
 
   // Comma-separated list of additional jars to add to the HMS classpath.
   string aux_jars = 
Substitute("$0/hms-plugin.jar,$1/hcatalog/share/hcatalog/*",
@@ -111,10 +112,6 @@ Status MiniHms::Start() {
   string java_options =
 // Ensure IPv4 is used.
 "-Djava.net.preferIPv4Stack=true "
-// Make logging less verbose.
-"-Dhive.log.level=WARN "
-// Log to the console.
-"-Dhive.root.logger=console "
 // Tune down the Derby deadlock timeout. The HMS's use of Derby with the
 // NOTIFICATION_SEQUENCE table is prone to deadlocks, at which point Derby
 // cancels a conflicting transaction after waiting out the timeout. This
@@ -321,5 +318,38 @@ Status MiniHms::CreateCoreSite() const {
JoinPathSegments(data_root_, "core-site.xml"));
 }
 
+Status MiniHms::CreateLogConfig() const {
+  // Configure the HMS to output ERROR messages to the stderr console, and INFO
+  // and above to hms.log in the data root. The console messages have a special
+  // 'HMS' tag included to disambiguate them from other Java component logs. 
The
+  // HMS automatically looks for a logging configuration named
+  // 'hive-log4j2.properties' in the configured HIVE_CONF_DIR.
+  static const string kFileTemplate = R"(
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} [HMS - %p - %t] (%F:%L) %m%n
+appender.console.filter.threshold.type = ThresholdFilter
+appender.console.filter.threshold.level = ERROR
+
+appender.file.type = File
+appender.file.name = file
+appender.file.fileName = $0
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} [%p - %t] (%F:%L) %m%n
+
+rootLogger.level = INFO
+rootLogger.appenderRefs = console, file
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.file.ref = file
+  )";
+
+  string file_contents = Substitute(kFileTemplate, 
JoinPathSegments(data_root_, "hms.log"));
+
+  return WriteStringToFile(Env::Default(),
+   file_contents,
+   JoinPathSegments(data_root_, 
"hive-log4j2.properties"));
+}
 } // namespace hms
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/61aed1a5/src/kudu/hms/mini_hms.h
--
diff --git a/src/kudu/hms/mini_hms.h b/src/kudu/hms/mini_hms.h
index 6528e0b..b98fa4b 100644
--- a/src/kudu/hms/mini_hms.h
+++ b/src/kudu/hms/mini_hms.h
@@ -86,6 +86,9 @@ class MiniHms {
   // Creates a core-site.xml for the mini HMS.
   Status CreateCoreSite() const WARN_UNUSED_RESULT;
 
+  // Creates a log4j2 configuration properties file for the mini HMS.
+  Status CreateLogConfig() const 

kudu git commit: Bump Sentry and Hadoop versions

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 2d6f8ffa9 -> 8d6cfe10d


Bump Sentry and Hadoop versions

The motivation is to get access to SENTRY-2371, which introduces a new
Thrift interface that will be used by Kudu to retrieve user privileges.
SENTRY-2371 has not been released yet, so the new Sentry version is
based on the latest commit of the master branch. The new Sentry version
appears to have an incompatibility with Hadoop 2.8.2, so this also
includes a bump to the latest Hadoop 2.8.x version. I suspect the fix
between 2.8.2 and 2.8.5 was introduced in [1], but the commit message is
vague and I haven't dug any further. The incompatiblity manifests as an
exception during Sentry startup:

java.lang.NoSuchMethodError: 
org.apache.hadoop.conf.Configuration.addResource(Ljava/net/URL;Z)V
at 
org.apache.sentry.service.thrift.SentryService.loadConfig(SentryService.java:576)
at 
org.apache.sentry.service.thrift.SentryService$CommandImpl.run(SentryService.java:600)
at org.apache.sentry.SentryMain.main(SentryMain.java:120)

[1]: 
https://github.com/apache/hadoop/commit/7af9b8ad1e993ef791aa38740b6aabc4c233a30f

Change-Id: I8bcc4ff6fac0435b037b984f45da75bed6ff4be5
Reviewed-on: http://gerrit.cloudera.org:8080/11601
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8d6cfe10
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8d6cfe10
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8d6cfe10

Branch: refs/heads/master
Commit: 8d6cfe10d47bdb422c6cc0e770afbd70d6ec7adf
Parents: 2d6f8ff
Author: Dan Burkert 
Authored: Fri Oct 5 14:10:32 2018 -0700
Committer: Dan Burkert 
Committed: Mon Oct 8 17:59:17 2018 +

--
 src/kudu/sentry/sentry_common_service.thrift |   3 +-
 src/kudu/sentry/sentry_policy_service.thrift | 100 --
 thirdparty/vars.sh   |   6 +-
 3 files changed, 99 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/src/kudu/sentry/sentry_common_service.thrift
--
diff --git a/src/kudu/sentry/sentry_common_service.thrift 
b/src/kudu/sentry/sentry_common_service.thrift
index e37f4f6..b8a730e 100644
--- a/src/kudu/sentry/sentry_common_service.thrift
+++ b/src/kudu/sentry/sentry_common_service.thrift
@@ -19,7 +19,7 @@
  */
 
 # DO NOT MODIFY! Copied from
-# 
https://raw.githubusercontent.com/apache/sentry/release-2.0.1/sentry-provider/sentry-provider-db/src/main/resources/sentry_common_service.thrift
+# 
https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
 #
 # With edits:
 #   - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -47,4 +47,3 @@ struct TSentryResponseStatus {
 2: required string message
 3: optional string stack
 }
-

http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/src/kudu/sentry/sentry_policy_service.thrift
--
diff --git a/src/kudu/sentry/sentry_policy_service.thrift 
b/src/kudu/sentry/sentry_policy_service.thrift
index a11872b..d32e39b 100644
--- a/src/kudu/sentry/sentry_policy_service.thrift
+++ b/src/kudu/sentry/sentry_policy_service.thrift
@@ -19,7 +19,7 @@
  */
 
 # DO NOT MODIFY! Copied from
-# 
https://raw.githubusercontent.com/apache/sentry/release-2.0.1/sentry-provider/sentry-provider-db/src/main/resources/sentry_policy_service.thrift
+# 
https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
 #
 # With edits:
 #   - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -30,8 +30,8 @@
 
 include "sentry_common_service.thrift"
 
-namespace java org.apache.sentry.provider.db.service.thrift
-namespace php sentry.provider.db.service.thrift
+namespace java org.apache.sentry.api.service.thrift
+namespace php sentry.api.service.thrift
 namespace cpp sentry
 
 enum TSentryGrantOption {
@@ -44,6 +44,12 @@ enum TSentryGrantOption {
   UNSET = -1
 }
 
+enum TSentryPrincipalType {
+  NONE = 0,
+  ROLE = 1,
+  USER = 2
+}
+
 # Represents a Privilege in transport from the client to the server
 struct TSentryPrivilege {
 1: required string privilegeScope, # Valid values are SERVER, DATABASE, TABLE, 
COLUMN, URI
@@ -190,9 +196,18 @@ struct TSentryAuthorizable {
 struct TListSentryPrivilegesRequest {
 1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
 2: required string requestorUserName, # user on whose behalf the request is 
issued
+
+# @Deprecated Use principalName instead to set role names or 

kudu git commit: Extract connection retrying and HA support from HmsCatalog

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master f1f192ca3 -> 42b75278a


Extract connection retrying and HA support from HmsCatalog

Introduces a new abstraction, thrift::HaClient, for providing HA and
retrying support on top of Thrift clients like HmsClient and
SentryClient. The implementation is extracted from HmsCatalog, and is
not changed in any significant ways.

The implementation is inlined into a header because the class requires a
template parameter (to choose between HmsClient or SentryClient), and
forward declaring these instantiations is not possible because the
thrift module does not link to the hms or sentry modules.

No tests are provided for HaClient in isolation, instead it's tested
through existing and new failover tests of HaClient and
HaClient.

This patch should have little or no functional changes.

Change-Id: Id8135f4c3995bba0ba28384c35696a1771ff5296
Reviewed-on: http://gerrit.cloudera.org:8080/11570
Reviewed-by: Adar Dembo 
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/42b75278
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/42b75278
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/42b75278

Branch: refs/heads/master
Commit: 42b75278a8d226b0235736aae735edeeeb55ab21
Parents: f1f192c
Author: Dan Burkert 
Authored: Tue Oct 2 14:05:00 2018 -0700
Committer: Dan Burkert 
Committed: Fri Oct 5 21:42:43 2018 +

--
 src/kudu/hms/hms_catalog.cc   | 216 
 src/kudu/hms/hms_catalog.h|  33 +---
 src/kudu/hms/hms_client.cc|   2 +
 src/kudu/hms/hms_client.h |  23 +--
 src/kudu/sentry/sentry_client-test.cc |  29 +++-
 src/kudu/sentry/sentry_client.cc  |   2 +
 src/kudu/sentry/sentry_client.h   |  13 +-
 src/kudu/thrift/client.cc |  16 +-
 src/kudu/thrift/client.h  | 256 -
 9 files changed, 343 insertions(+), 247 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/42b75278/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index ae4d998..7620b2e 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -17,8 +17,6 @@
 
 #include "kudu/hms/hms_catalog.h"
 
-#include 
-#include 
 #include 
 #include 
 #include 
@@ -43,11 +41,10 @@
 #include "kudu/hms/hive_metastore_types.h"
 #include "kudu/hms/hms_client.h"
 #include "kudu/thrift/client.h"
-#include "kudu/util/async_util.h"
 #include "kudu/util/flag_tags.h"
+#include "kudu/util/monotime.h"
 #include "kudu/util/net/net_util.h"
 #include "kudu/util/slice.h"
-#include "kudu/util/threadpool.h"
 
 using boost::none;
 using boost::optional;
@@ -123,12 +120,7 @@ const char* const HmsCatalog::kInvalidTableError = "when 
the Hive Metastore inte
 "identifier pair, each containing only ASCII alphanumeric characters, '_', 
and '/'";
 
 HmsCatalog::HmsCatalog(string master_addresses)
-: master_addresses_(std::move(master_addresses)),
-  hms_client_(HostPort("", 0), thrift::ClientOptions()),
-  reconnect_after_(MonoTime::Now()),
-  reconnect_failure_(Status::OK()),
-  consecutive_reconnect_failures_(0),
-  reconnect_idx_(0) {
+: master_addresses_(std::move(master_addresses)) {
 }
 
 HmsCatalog::~HmsCatalog() {
@@ -136,35 +128,32 @@ HmsCatalog::~HmsCatalog() {
 }
 
 Status HmsCatalog::Start() {
-  if (threadpool_) {
-return Status::IllegalState("HMS Catalog is already started");
-  }
-
-  RETURN_NOT_OK(ParseUris(FLAGS_hive_metastore_uris, _addresses_));
+  vector addresses;
+  RETURN_NOT_OK(ParseUris(FLAGS_hive_metastore_uris, ));
 
-  // The thread pool must be capped at one thread to ensure serialized access 
to
-  // the fields of HmsCatalog.
-  RETURN_NOT_OK(ThreadPoolBuilder("hms-catalog")
-  .set_min_threads(1)
-  .set_max_threads(1)
-  .Build(_));
+  thrift::ClientOptions options;
+  options.send_timeout = 
MonoDelta::FromSeconds(FLAGS_hive_metastore_send_timeout);
+  options.recv_timeout = 
MonoDelta::FromSeconds(FLAGS_hive_metastore_recv_timeout);
+  options.conn_timeout = 
MonoDelta::FromSeconds(FLAGS_hive_metastore_conn_timeout);
+  options.enable_kerberos = FLAGS_hive_metastore_sasl_enabled;
+  options.service_principal = FLAGS_hive_metastore_kerberos_principal;
+  options.max_buf_size = FLAGS_hive_metastore_max_message_size;
+  options.retry_count = FLAGS_hive_metastore_retry_count;
 
-  return Status::OK();
+  return ha_client_.Start(std::move(addresses), std::move(options));
 }
 
 void HmsCatalog::Stop() {
-  if (threadpool_) {
-threadpool_->Shutdown();
-  }
+  ha_client_.Stop();
 }
 
 Status HmsCatalog::CreateTable(const string& id,
  

[3/4] kudu git commit: KUDU-2541: Fill out basic sentry client API

[danburkert]

KUDU-2541: Fill out basic sentry client API

This commit contains more plumbing for the upcoming Sentry integration
(KUDU-428). The Sentry client class is filled out with enough methods to
ensure the Thrift RPC is working correctly, as well as error conversion
from Sentry and Thrift's format to Kudu's Status.

Change-Id: I1004a11506ed109d1f23389ca73d95a34c32c194
Reviewed-on: http://gerrit.cloudera.org:8080/11524
Reviewed-by: Adar Dembo 
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/14f3e6f6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/14f3e6f6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/14f3e6f6

Branch: refs/heads/master
Commit: 14f3e6f60b9fbfc7c7b41c6b5ce4ba8612088869
Parents: fad56a6
Author: Dan Burkert 
Authored: Fri Sep 21 10:43:05 2018 -0700
Committer: Dan Burkert 
Committed: Thu Sep 27 00:34:24 2018 +

--
 src/kudu/sentry/CMakeLists.txt|   1 +
 src/kudu/sentry/mini_sentry.cc|  53 ++--
 src/kudu/sentry/mini_sentry.h |   5 +-
 src/kudu/sentry/sentry_client-test.cc |  57 +
 src/kudu/sentry/sentry_client.cc  | 131 +
 src/kudu/sentry/sentry_client.h   |  63 ++
 6 files changed, 300 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/14f3e6f6/src/kudu/sentry/CMakeLists.txt
--
diff --git a/src/kudu/sentry/CMakeLists.txt b/src/kudu/sentry/CMakeLists.txt
index 8e96c97..910117c 100644
--- a/src/kudu/sentry/CMakeLists.txt
+++ b/src/kudu/sentry/CMakeLists.txt
@@ -35,6 +35,7 @@ set(SENTRY_SRCS
   sentry_client.cc)
 set(SENTRY_DEPS
   kudu_common
+  kudu_thrift
   kudu_util
   sentry_thrift)
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/14f3e6f6/src/kudu/sentry/mini_sentry.cc
--
diff --git a/src/kudu/sentry/mini_sentry.cc b/src/kudu/sentry/mini_sentry.cc
index 865fd2d..ff54ad8 100644
--- a/src/kudu/sentry/mini_sentry.cc
+++ b/src/kudu/sentry/mini_sentry.cc
@@ -74,7 +74,7 @@ Status MiniSentry::Start() {
 
   auto tmp_dir = GetTestDataDirectory();
 
-  RETURN_NOT_OK(CreateSentrySite(tmp_dir));
+  RETURN_NOT_OK(CreateSentryConfigs(tmp_dir));
 
   map env_vars {
   { "JAVA_HOME", java_home },
@@ -126,7 +126,7 @@ Status MiniSentry::Resume() {
   return Status::OK();
 }
 
-Status MiniSentry::CreateSentrySite(const string& tmp_dir) const {
+Status MiniSentry::CreateSentryConfigs(const string& tmp_dir) const {
 
   // - sentry.store.jdbc.url
   // - sentry.store.jdbc.password
@@ -136,6 +136,15 @@ Status MiniSentry::CreateSentrySite(const string& tmp_dir) 
const {
   // - datanucleus.schema.autoCreateAll
   // - sentry.verify.schema.version
   // Allow Sentry to startup and run without first running the schemaTool.
+  //
+  // - sentry.store.group.mapping
+  //   sentry.store.group.mapping.resource
+  // Production Sentry instances use Hadoop's UGI infrastructure to map 
users
+  // to groups, but that's difficult to mock for tests, so we configure a
+  // simpler static user/group mapping based on a generated INI file.
+  //
+  // - sentry.service.admin.group
+  // Set up admin groups which have unrestricted privileges in Sentry.
   static const string kFileTemplate = R"(
 
 
@@ -160,17 +169,45 @@ Status MiniSentry::CreateSentrySite(const string& 
tmp_dir) const {
   
 
   
-   sentry.verify.schema.version
+sentry.verify.schema.version
 false
   
+
+  
+sentry.store.group.mapping
+org.apache.sentry.provider.file.LocalGroupMappingService
+  
+
+  
+sentry.store.group.mapping.resource
+$1
+  
+
+  
+sentry.service.admin.group
+admin
+  
 
   )";
 
-  string file_contents = Substitute(kFileTemplate, tmp_dir);
-
-  return WriteStringToFile(Env::Default(),
-   file_contents,
-   JoinPathSegments(tmp_dir, "sentry-site.xml"));
+  string users_ini_path = JoinPathSegments(tmp_dir, "users.ini");
+  string file_contents = Substitute(kFileTemplate, tmp_dir, users_ini_path);
+  RETURN_NOT_OK(WriteStringToFile(Env::Default(),
+  file_contents,
+  JoinPathSegments(tmp_dir, 
"sentry-site.xml")));
+
+  // Simple file format containing mapping of user to groups in INI syntax, see
+  // the LocalGroupMappingService class for more information.
+  static const string kUsers = R"(
+[users]
+test-admin=admin
+test-user=user
+kudu=admin
+joe-interloper=""
+)";
+
+  RETURN_NOT_OK(WriteStringToFile(Env::Default(), kUsers, users_ini_path));
+  return Status::OK();
 }
 
 } // namespace sentry

[4/4] kudu git commit: KUDU-2541: add Kerberos support to Sentry client and mini-cluster

[danburkert]

KUDU-2541: add Kerberos support to Sentry client and mini-cluster

More infrastructure work towards KUDU-428.

Change-Id: I3f7e27137135cb9b463a90b98e4aba864cece3c1
Reviewed-on: http://gerrit.cloudera.org:8080/11525
Reviewed-by: Hao Hao 
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ecc4998c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ecc4998c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ecc4998c

Branch: refs/heads/master
Commit: ecc4998cb921b97d919ce6acef2b0f623a75d653
Parents: 14f3e6f
Author: Dan Burkert 
Authored: Mon Sep 24 15:47:39 2018 -0700
Committer: Dan Burkert 
Committed: Thu Sep 27 00:34:32 2018 +

--
 src/kudu/sentry/CMakeLists.txt|  1 +
 src/kudu/sentry/mini_sentry.cc| 53 +++---
 src/kudu/sentry/mini_sentry.h | 10 ++
 src/kudu/sentry/sentry_client-test.cc | 41 +++
 4 files changed, 95 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ecc4998c/src/kudu/sentry/CMakeLists.txt
--
diff --git a/src/kudu/sentry/CMakeLists.txt b/src/kudu/sentry/CMakeLists.txt
index 910117c..588bf9f 100644
--- a/src/kudu/sentry/CMakeLists.txt
+++ b/src/kudu/sentry/CMakeLists.txt
@@ -72,6 +72,7 @@ target_link_libraries(mini_sentry
 if (NOT NO_TESTS)
   SET_KUDU_TEST_LINK_LIBS(
 kudu_sentry
+mini_kdc
 mini_sentry)
 
   ADD_KUDU_TEST(sentry_client-test)

http://git-wip-us.apache.org/repos/asf/kudu/blob/ecc4998c/src/kudu/sentry/mini_sentry.cc
--
diff --git a/src/kudu/sentry/mini_sentry.cc b/src/kudu/sentry/mini_sentry.cc
index ff54ad8..f50aa5b 100644
--- a/src/kudu/sentry/mini_sentry.cc
+++ b/src/kudu/sentry/mini_sentry.cc
@@ -53,6 +53,18 @@ MiniSentry::~MiniSentry() {
   WARN_NOT_OK(Stop(), "Failed to stop MiniSentry");
 }
 
+void MiniSentry::EnableKerberos(std::string krb5_conf,
+std::string service_principal,
+std::string keytab_file) {
+  CHECK(!sentry_process_);
+  CHECK(!krb5_conf.empty());
+  CHECK(!service_principal.empty());
+  CHECK(!keytab_file.empty());
+  krb5_conf_ = std::move(krb5_conf);
+  service_principal_ = std::move(service_principal);
+  keytab_file_ = std::move(keytab_file);
+}
+
 Status MiniSentry::Start() {
   SCOPED_LOG_SLOW_EXECUTION(WARNING, kSentryStartTimeoutMs / 2, "Starting 
Sentry");
   CHECK(!sentry_process_);
@@ -76,9 +88,16 @@ Status MiniSentry::Start() {
 
   RETURN_NOT_OK(CreateSentryConfigs(tmp_dir));
 
+  // List of JVM environment options to pass to the Sentry service.
+  string java_options;
+  if (!krb5_conf_.empty()) {
+java_options += Substitute(" -Djava.security.krb5.conf=$0", krb5_conf_);
+  }
+
   map env_vars {
   { "JAVA_HOME", java_home },
   { "HADOOP_HOME", hadoop_home },
+  { "JAVA_TOOL_OPTIONS", java_options },
   };
 
   // Start Sentry.
@@ -145,17 +164,31 @@ Status MiniSentry::CreateSentryConfigs(const string& 
tmp_dir) const {
   //
   // - sentry.service.admin.group
   // Set up admin groups which have unrestricted privileges in Sentry.
+  //
+  // - sentry.service.allow.connect
+  // Set of Kerberos principals which is allowed to connect to Sentry when
+  // the Kerberos security mode is enabled.
   static const string kFileTemplate = R"(
 
 
   
 sentry.service.security.mode
-none
+$0
+  
+
+  
+sentry.service.server.principal
+$1
+  
+
+  
+sentry.service.server.keytab
+$2
   
 
   
 sentry.store.jdbc.url
-jdbc:derby:$0/sentry;create=true
+jdbc:derby:$3/sentry;create=true
   
 
   
@@ -180,18 +213,30 @@ Status MiniSentry::CreateSentryConfigs(const string& 
tmp_dir) const {
 
   
 sentry.store.group.mapping.resource
-$1
+$4
   
 
   
 sentry.service.admin.group
 admin
   
+
+  
+sentry.service.allow.connect
+kudu
+  
+
 
   )";
 
   string users_ini_path = JoinPathSegments(tmp_dir, "users.ini");
-  string file_contents = Substitute(kFileTemplate, tmp_dir, users_ini_path);
+  string file_contents = Substitute(
+  kFileTemplate,
+  keytab_file_.empty() ? "none" : "kerberos",
+  service_principal_,
+  keytab_file_,
+  tmp_dir,
+  users_ini_path);
   RETURN_NOT_OK(WriteStringToFile(Env::Default(),
   file_contents,
   JoinPathSegments(tmp_dir, 
"sentry-site.xml")));

http://git-wip-us.apache.org/repos/asf/kudu/blob/ecc4998c/src/kudu/sentry/mini_sentry.h
--
diff --git a/src/kudu/sentry/mini_sentry.h b/src/kudu/sentry/mini_sentry.h

[1/4] kudu git commit: [DOCS] Added a version notice for changing managed Kudu table names in Impala

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master c9f1772e7 -> ecc4998cb


[DOCS] Added a version notice for changing managed Kudu table names in Impala

With IMPALA-5654, users can no longer change kudu.table_name property
for managed Kudu tables in Impala.

Change-Id: If26f103d931dd7ed57b56cf34b0010d14c098928
Reviewed-on: http://gerrit.cloudera.org:8080/11515
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bfdcd973
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bfdcd973
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bfdcd973

Branch: refs/heads/master
Commit: bfdcd973f2338f1326456cccdb62fbee9530ece5
Parents: c9f1772
Author: Alex Rodoni 
Authored: Tue Sep 25 15:19:36 2018 -0700
Committer: Alexey Serbin 
Committed: Wed Sep 26 22:28:31 2018 +

--
 docs/kudu_impala_integration.adoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bfdcd973/docs/kudu_impala_integration.adoc
--
diff --git a/docs/kudu_impala_integration.adoc 
b/docs/kudu_impala_integration.adoc
index f16711d..2fe7e27 100755
--- a/docs/kudu_impala_integration.adoc
+++ b/docs/kudu_impala_integration.adoc
@@ -683,8 +683,8 @@ underlying Kudu table.
 
 .Rename the underlying Kudu table for an internal table
 
-If a table is an internal table, the underlying Kudu table may be renamed by
-changing the `kudu.table_name` property:
+In Impala 2.11 and lower, the underlying Kudu table may be renamed by changing
+the `kudu.table_name` property:
 
 [source,sql]
 

[2/4] kudu git commit: KUDU-428: add Sentry to thirdparty, mini-sentry

[danburkert]

KUDU-428: add Sentry to thirdparty, mini-sentry

This commit adds Sentry to thirdparty, and fills out the MiniSentry
class with an initial implementation. Notable features that aren't
implemented:

- Stripped Sentry packaging. I've put an unmodified version of Sentry
  2.0.1 into thirdparty. It weighs in at almost 200MiB and takes about
  5s to startup on my laptop. We will probably want to add a stripped
  version later to reduce both of these.

- Kerberos support for mini-sentry. Right now Kerberos is disabled,
  which is an atypical configuration. A follow-up commit will add a
  Kerberos support configuration taking advantage of the mini KDC.

- The mini Sentry is not yet configured with the location of the HMS,
  which will be necessary to do anything non-trivial with it.

Change-Id: I03f39cf9b2c813c0c305d085e1ad3851636326f5
Reviewed-on: http://gerrit.cloudera.org:8080/11347
Reviewed-by: Adar Dembo 
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fad56a6f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fad56a6f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fad56a6f

Branch: refs/heads/master
Commit: fad56a6f85a4707f075f2334de21fb9025011795
Parents: bfdcd97
Author: Dan Burkert 
Authored: Thu Aug 23 16:40:24 2018 -0700
Committer: Dan Burkert 
Committed: Thu Sep 27 00:34:17 2018 +

--
 build-support/dist_test.py|   9 +-
 build-support/run_dist_test.py|   3 +-
 src/kudu/hms/mini_hms.cc  |  37 ++-
 src/kudu/sentry/CMakeLists.txt|  10 ++
 src/kudu/sentry/mini_sentry.cc| 154 +
 src/kudu/sentry/mini_sentry.h |  47 +
 src/kudu/sentry/sentry_client-test.cc |  19 ++--
 src/kudu/util/test_util.cc|  16 +++
 src/kudu/util/test_util.h |   9 ++
 thirdparty/build-thirdparty.sh|  10 +-
 thirdparty/download-thirdparty.sh |   6 ++
 thirdparty/vars.sh|   4 +
 12 files changed, 281 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fad56a6f/build-support/dist_test.py
--
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index c1176fe..b2e7db9 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -97,12 +97,13 @@ DEPS_FOR_ALL = \
  # Tests that require tooling require this.
  "build/latest/bin/kudu",
 
- # The HMS tests require the Hadoop and Hive libraries. These files are 
just
- # symlinks, but dist-test will copy the entire directories they point to.
- # The symlinks themselves won't be recreated, so we point to them with
- # environment variables in run_dist_test.py.
+ # The HMS and Sentry tests require the Hadoop, Hive, and Sentry libraries.
+ # These files are just symlinks, but dist-test will copy the entire
+ # directories they point to.  The symlinks themselves won't be recreated,
+ # so we point to them with environment variables in run_dist_test.py.
  "build/latest/bin/hive-home",
  "build/latest/bin/hadoop-home",
+ "build/latest/bin/sentry-home",
 
  # Add the Kudu HMS plugin.
  "build/latest/bin/hms-plugin.jar",

http://git-wip-us.apache.org/repos/asf/kudu/blob/fad56a6f/build-support/run_dist_test.py
--
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index 297075d..1edcce2 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -147,9 +147,10 @@ def main():
   env[var_name] = os.environ.get(var_name, "") + " 
external_symbolizer_path=" + symbolizer_path
 
   # Add environment variables for Java dependencies. These environment 
variables
-  # are used in mini_hms.cc.
+  # are used in mini_hms.cc and mini_sentry.cc.
   env['HIVE_HOME'] = glob.glob(os.path.join(ROOT, "thirdparty/src/hive-*"))[0]
   env['HADOOP_HOME'] = glob.glob(os.path.join(ROOT, 
"thirdparty/src/hadoop-*"))[0]
+  env['SENTRY_HOME'] = glob.glob(os.path.join(ROOT, 
"thirdparty/src/apache-sentry-*"))[0]
   env['JAVA_HOME'] = glob.glob("/usr/lib/jvm/java-1.8.0-*")[0]
 
   env['LD_LIBRARY_PATH'] = ":".join(

http://git-wip-us.apache.org/repos/asf/kudu/blob/fad56a6f/src/kudu/hms/mini_hms.cc
--
diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc
index cc7e6bc..253de09 100644
--- a/src/kudu/hms/mini_hms.cc
+++ b/src/kudu/hms/mini_hms.cc
@@ -19,7 +19,6 @@
 
 #include 
 #include 
-#include 
 #include 
 #include 
 #include 
@@ -34,7 +33,6 @@
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 #include "kudu/util/stopwatch.h"

[2/2] kudu git commit: Add Hive Metastore service principal configuration

[danburkert]

Add Hive Metastore service principal configuration

This commit adds a new flag, --hive_metastore_kerberos_principal flag
which corresponds to the HMS 'hive.metastore.kerberos.principal'
configuration. This configuration is rarely overridden, but in cases
where it is, having a way to match it in Kudu is critical.

Change-Id: I6c5c56423a62cba0b696f97d866077b35845ce26
Reviewed-on: http://gerrit.cloudera.org:8080/11503
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a09c89ab
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a09c89ab
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a09c89ab

Branch: refs/heads/master
Commit: a09c89abc1c0df62cffc9bdf3b36eede9150c4b9
Parents: 24b38f7
Author: Dan Burkert 
Authored: Mon Sep 24 16:41:27 2018 -0700
Committer: Dan Burkert 
Committed: Tue Sep 25 17:42:49 2018 +

--
 src/kudu/hms/hms_catalog-test.cc| 1 +
 src/kudu/hms/hms_catalog.cc | 7 +++
 src/kudu/hms/hms_client-test.cc | 5 -
 src/kudu/integration-tests/master_hms-itest.cc  | 1 +
 src/kudu/mini-cluster/external_mini_cluster-test.cc | 1 +
 src/kudu/thrift/client.cc   | 4 +++-
 src/kudu/thrift/client.h| 6 ++
 src/kudu/thrift/sasl_client_transport.cc| 9 +
 src/kudu/thrift/sasl_client_transport.h | 6 +-
 src/kudu/tools/kudu-tool-test.cc| 3 +++
 10 files changed, 36 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a09c89ab/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index a2bfabf..83369fb 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -184,6 +184,7 @@ class HmsCatalogTest : public KuduTest {
   ASSERT_OK(kdc_->Kinit("alice"));
   ASSERT_OK(kdc_->SetKrb5Environment());
   hms_client_opts.enable_kerberos = true;
+  hms_client_opts.service_principal = "hive";
 
   // Configure the HmsCatalog flags.
   FLAGS_hive_metastore_sasl_enabled = true;

http://git-wip-us.apache.org/repos/asf/kudu/blob/a09c89ab/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index aeaf226..ae4d998 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -73,6 +73,12 @@ DEFINE_bool(hive_metastore_sasl_enabled, false,
 "When enabled, the --keytab_file flag must be provided.");
 TAG_FLAG(hive_metastore_sasl_enabled, experimental);
 
+DEFINE_string(hive_metastore_kerberos_principal, "hive",
+  "The service principal of the Hive Metastore server. Must match "
+  "the primary (user) portion of hive.metastore.kerberos.principal 
option "
+  "in the Hive Metastore configuration.");
+TAG_FLAG(hive_metastore_kerberos_principal, experimental);
+
 DEFINE_int32(hive_metastore_retry_count, 1,
  "The number of times that HMS operations will retry after "
  "encountering retriable failures, such as network errors.");
@@ -416,6 +422,7 @@ Status HmsCatalog::Reconnect() {
   options.recv_timeout = 
MonoDelta::FromSeconds(FLAGS_hive_metastore_recv_timeout);
   options.conn_timeout = 
MonoDelta::FromSeconds(FLAGS_hive_metastore_conn_timeout);
   options.enable_kerberos = FLAGS_hive_metastore_sasl_enabled;
+  options.service_principal = FLAGS_hive_metastore_kerberos_principal;
   options.max_buf_size = FLAGS_hive_metastore_max_message_size;
 
   // Try reconnecting to each HMS in sequence, returning the first one which

http://git-wip-us.apache.org/repos/asf/kudu/blob/a09c89ab/src/kudu/hms/hms_client-test.cc
--
diff --git a/src/kudu/hms/hms_client-test.cc b/src/kudu/hms/hms_client-test.cc
index e677cd0..9091a46 100644
--- a/src/kudu/hms/hms_client-test.cc
+++ b/src/kudu/hms/hms_client-test.cc
@@ -127,6 +127,7 @@ TEST_P(HmsClientTest, TestHmsOperations) {
 ASSERT_OK(kdc.Kinit("alice"));
 ASSERT_OK(kdc.SetKrb5Environment());
 hms_client_opts.enable_kerberos = true;
+hms_client_opts.service_principal = "hive";
   }
 
   ASSERT_OK(hms.Start());
@@ -288,7 +289,8 @@ TEST_P(HmsClientTest, TestLargeObjects) {
   if (protection) {
 ASSERT_OK(kdc.Start());
 
-string spn = "hive/127.0.0.1";
+// Try a non-standard service principal to ensure it works correctly.
+string spn = "hive_alternate_sp/127.0.0.1";
 string ktpath;
 ASSERT_OK(kdc.CreateServiceKeytab(spn, ));
 
@@ -302,6 

[1/2] kudu git commit: hybrid_clock: restore SleepFor in WaitUntilAfterLocally

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master ddaa766e2 -> a09c89abc


hybrid_clock: restore SleepFor in WaitUntilAfterLocally

This seems to have gone missing from commit f2d96437c. The end result is a
busy loop, which was probably unintentional. I also removed the loop, which
seemed unnecessary with the SleepFor.

To test, I wrote a small program that scanned a table at Now() + 10s, after
setting --scanner_max_wait_ms to 1 (default is 1000). Although the wall
clock time was the same, the cycle count was far higher per perf stat.

Change-Id: Iecbb077b1f88293dcb0ab53e40ff3862f772694e
Reviewed-on: http://gerrit.cloudera.org:8080/11495
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/24b38f73
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/24b38f73
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/24b38f73

Branch: refs/heads/master
Commit: 24b38f73739ef76f69fbe950ff103dc6c1b3e6a8
Parents: ddaa766
Author: Adar Dembo 
Authored: Fri Sep 21 12:27:41 2018 -0700
Committer: David Ribeiro Alves 
Committed: Mon Sep 24 15:03:30 2018 +

--
 src/kudu/clock/hybrid_clock.cc | 32 
 1 file changed, 16 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/24b38f73/src/kudu/clock/hybrid_clock.cc
--
diff --git a/src/kudu/clock/hybrid_clock.cc b/src/kudu/clock/hybrid_clock.cc
index b97fd2d..7e87d4d 100644
--- a/src/kudu/clock/hybrid_clock.cc
+++ b/src/kudu/clock/hybrid_clock.cc
@@ -291,31 +291,31 @@ Status HybridClock::WaitUntilAfter(const Timestamp& then,
 
   SleepFor(MonoDelta::FromMicroseconds(wait_for_usec));
 
-
   VLOG(1) << "WaitUntilAfter(): Incoming time(latest): " << then_latest_usec
   << " Now(earliest): " << now_earliest_usec << " error: " << error
   << " Waiting for: " << wait_for_usec;
-
   return Status::OK();
 }
 
 Status HybridClock::WaitUntilAfterLocally(const Timestamp& then,
   const MonoTime& deadline) {
-  while (true) {
-Timestamp now;
-uint64_t error;
-{
-  std::lock_guard lock(lock_);
-  NowWithError(, );
-}
-if (now > then) {
-  return Status::OK();
-}
-uint64_t wait_for_usec = GetPhysicalValueMicros(then) - 
GetPhysicalValueMicros(now);
-
-// Check that sleeping wouldn't sleep longer than our deadline.
-RETURN_NOT_OK(CheckDeadlineNotWithinMicros(deadline, wait_for_usec));
+  Timestamp now;
+  uint64_t error;
+  {
+std::lock_guard lock(lock_);
+NowWithError(, );
   }
+  if (now > then) {
+return Status::OK();
+  }
+  uint64_t wait_for_usec = GetPhysicalValueMicros(then) - 
GetPhysicalValueMicros(now);
+
+  // Check that sleeping wouldn't sleep longer than our deadline.
+  RETURN_NOT_OK(CheckDeadlineNotWithinMicros(deadline, wait_for_usec));
+
+  SleepFor(MonoDelta::FromMicroseconds(wait_for_usec));
+
+  return Status::OK();
 }
 
 bool HybridClock::IsAfter(Timestamp t) {

kudu git commit: Add thrift module for common thrift utilities

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 6129467d9 -> ddaa766e2


Add thrift module for common thrift utilities

The HMS patch series inlined all necessary Thrift utilities into the hms
module, since it was the only use of Thrift in the codebase. Now that
we're also planning on having a Sentry client it makes sense to properly
abstract the common Thrift code into its own shared module.

Change-Id: I6f6f843f42b37cb1170df03da01fc0790fe94acb
Reviewed-on: http://gerrit.cloudera.org:8080/11493
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ddaa766e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ddaa766e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ddaa766e

Branch: refs/heads/master
Commit: ddaa766e2fdf177a86ed3eb57e6b917730b75843
Parents: 6129467
Author: Dan Burkert 
Authored: Fri Sep 21 12:40:18 2018 -0700
Committer: Dan Burkert 
Committed: Sat Sep 22 00:39:23 2018 +

--
 CMakeLists.txt  |   1 +
 src/kudu/hms/CMakeLists.txt |   4 +-
 src/kudu/hms/hms_catalog-test.cc|   3 +-
 src/kudu/hms/hms_catalog.cc |  14 +-
 src/kudu/hms/hms_catalog.h  |   2 -
 src/kudu/hms/hms_client-test.cc |  11 +-
 src/kudu/hms/hms_client.cc  |  66 +--
 src/kudu/hms/hms_client.h   |  27 +-
 src/kudu/hms/sasl_client_transport.cc   | 402 ---
 src/kudu/hms/sasl_client_transport.h| 176 
 src/kudu/integration-tests/master_hms-itest.cc  |   4 +-
 .../mini-cluster/external_mini_cluster-test.cc  |   3 +-
 src/kudu/thrift/CMakeLists.txt  |  33 ++
 src/kudu/thrift/client.cc   |  85 
 src/kudu/thrift/client.h|  65 +++
 src/kudu/thrift/sasl_client_transport.cc| 402 +++
 src/kudu/thrift/sasl_client_transport.h | 176 
 src/kudu/tools/kudu-tool-test.cc|  10 +-
 18 files changed, 806 insertions(+), 678 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6c9d35e..df02893 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1477,6 +1477,7 @@ add_subdirectory(src/kudu/security)
 add_subdirectory(src/kudu/sentry)
 add_subdirectory(src/kudu/server)
 add_subdirectory(src/kudu/tablet)
+add_subdirectory(src/kudu/thrift)
 add_subdirectory(src/kudu/tools)
 add_subdirectory(src/kudu/tserver)
 add_subdirectory(src/kudu/util)

http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/src/kudu/hms/CMakeLists.txt
--
diff --git a/src/kudu/hms/CMakeLists.txt b/src/kudu/hms/CMakeLists.txt
index 90794a6..287307d 100644
--- a/src/kudu/hms/CMakeLists.txt
+++ b/src/kudu/hms/CMakeLists.txt
@@ -34,14 +34,14 @@ add_dependencies(hms_thrift ${HMS_THRIFT_TGTS})
 
 set(HMS_SRCS
   hms_catalog.cc
-  hms_client.cc
-  sasl_client_transport.cc)
+  hms_client.cc)
 set(HMS_DEPS
   gflags
   glog
   hms_thrift
   krpc
   kudu_common
+  kudu_thrift
   kudu_util)
 
 add_library(kudu_hms ${HMS_SRCS})

http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index cf9977b..a2bfabf 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -37,6 +37,7 @@
 #include "kudu/hms/mini_hms.h"
 #include "kudu/rpc/sasl_common.h"
 #include "kudu/security/test/mini_kdc.h"
+#include "kudu/thrift/client.h"
 #include "kudu/util/net/net_util.h"
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
@@ -161,7 +162,7 @@ class HmsCatalogTest : public KuduTest {
   void SetUp() override {
 bool enable_kerberos = EnableKerberos();
 
-HmsClientOptions hms_client_opts;
+thrift::ClientOptions hms_client_opts;
 
 hms_.reset(new hms::MiniHms());
 if (enable_kerberos) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index f62f9ba..aeaf226 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -42,6 +42,7 @@
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/hms/hive_metastore_types.h"
 #include "kudu/hms/hms_client.h"
+#include "kudu/thrift/client.h"
 #include "kudu/util/async_util.h"
 #include "kudu/util/flag_tags.h"
 #include 

kudu git commit: HMS integration: set table owner field in HMS table metadata

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master ed60c11a3 -> 97dbffb95


HMS integration: set table owner field in HMS table metadata

Other systems use table ownership for purposes like assigning
privileges. This patch sets the owner field in the HMS for Kudu tables
to the client's user name. A follow-up patch will add additional APIs to
the client CreateTable builders which will allow clients to override the
owner, for situations in which the client is actually proxying through
the table creation on behalf of a different user.

The HMS fix tool will create HMS table metadata for Kudu tables which
are missing it. This replacement table metadata will omit the table
owner, since it can't be reconstructed just from the Kudu table
metadata. This is the conservative approach, if we want to extend the
tool to allow passing in the owner as a flag we can do that in the
future.

Change-Id: I1a25aa0bb52bdd28df28a078fe91f55db9e29482
Reviewed-on: http://gerrit.cloudera.org:8080/11398
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/97dbffb9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/97dbffb9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/97dbffb9

Branch: refs/heads/master
Commit: 97dbffb95228b29b35ecba4be6708a33522ec152
Parents: ed60c11
Author: Dan Burkert 
Authored: Thu Sep 6 16:08:41 2018 -0700
Committer: Dan Burkert 
Committed: Tue Sep 11 22:45:28 2018 +

--
 src/kudu/hms/hms_catalog-test.cc   | 46 -
 src/kudu/hms/hms_catalog.cc| 18 +++--
 src/kudu/hms/hms_catalog.h |  9 +++
 src/kudu/integration-tests/master_hms-itest.cc | 46 +++--
 src/kudu/master/catalog_manager.cc |  3 +-
 src/kudu/tools/kudu-tool-test.cc   | 74 ++---
 src/kudu/tools/tool_action_hms.cc  |  4 +-
 7 files changed, 145 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/97dbffb9/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 4574402..cf9977b 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -24,6 +24,7 @@
 #include 
 #include 
 
+#include 
 #include 
 #include 
 
@@ -236,10 +237,16 @@ class HmsCatalogTest : public KuduTest {
   void CheckTable(const string& database_name,
   const string& table_name,
   const string& table_id,
+  const boost::optional& owner,
   const Schema& schema) {
 hive::Table table;
 ASSERT_OK(hms_client_->GetTable(database_name, table_name, ));
 
+if (owner) {
+  EXPECT_EQ(table.owner, *owner);
+} else {
+  EXPECT_TRUE(table.owner.empty());
+}
 EXPECT_EQ(table.parameters[HmsClient::kKuduTableIdKey], table_id);
 EXPECT_EQ(table.parameters[HmsClient::kKuduMasterAddrsKey], kMasterAddrs);
 EXPECT_EQ(table.parameters[HmsClient::kStorageHandlerKey], 
HmsClient::kKuduStorageHandler);
@@ -313,17 +320,18 @@ TEST_P(HmsCatalogTestParameterized, TestTableLifecycle) {
   const string kTableName = Substitute("$0.$1", kHmsDatabase, kHmsTableName);
   const string kHmsAlteredTableName = "altered_table_name";
   const string kAlteredTableName = Substitute("$0.$1", kHmsDatabase, 
kHmsAlteredTableName);
+  const string kOwner = "alice";
 
   Schema schema = AllTypesSchema();
 
   // Create the table.
-  ASSERT_OK(hms_catalog_->CreateTable(kTableId, kTableName, schema));
-  NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, schema));
+  ASSERT_OK(hms_catalog_->CreateTable(kTableId, kTableName, kOwner, schema));
+  NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, kOwner, schema));
 
   // Create the table again, and check that the expected failure occurs.
-  Status s = hms_catalog_->CreateTable(kTableId, kTableName, schema);
+  Status s = hms_catalog_->CreateTable(kTableId, kTableName, kOwner, schema);
   ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString();
-  NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, schema));
+  NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, kOwner, schema));
 
   // Alter the table.
   SchemaBuilder b(schema);
@@ -331,7 +339,7 @@ TEST_P(HmsCatalogTestParameterized, TestTableLifecycle) {
   Schema altered_schema = b.Build();
   ASSERT_OK(hms_catalog_->AlterTable(kTableId, kTableName, kAlteredTableName, 
altered_schema));
   NO_FATALS(CheckTableDoesNotExist(kHmsDatabase, kHmsTableName));
-  NO_FATALS(CheckTable(kHmsDatabase, kHmsAlteredTableName, kTableId, 
altered_schema));
+  NO_FATALS(CheckTable(kHmsDatabase, kHmsAlteredTableName, kTableId, kOwner, 

[1/3] kudu git commit: [master] fix compilation warning of DCHECK_NOTNULL

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master f0716cc52 -> ed60c11a3


[master] fix compilation warning of DCHECK_NOTNULL

Fixed compilation warning in case of RELEASE build:

kudu/master/ts_descriptor.cc:103:18: warning:
expression result unused [-Wunused-value]
  DCHECK_NOTNULL(location);
 ^~~~

This patch does not contain any functional changes.

This is a follow-up to dcc39d53d8c36a3f4896ce4c208b855abee8da83.

Change-Id: I768afd0ec34f8127f66ea8ca29ec337488c5
Reviewed-on: http://gerrit.cloudera.org:8080/11421
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7856ec91
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7856ec91
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7856ec91

Branch: refs/heads/master
Commit: 7856ec9174c16c212ce944983124954fe4d325fa
Parents: f0716cc
Author: Alexey Serbin 
Authored: Tue Sep 11 13:01:17 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Sep 11 20:35:18 2018 +

--
 src/kudu/master/ts_descriptor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7856ec91/src/kudu/master/ts_descriptor.cc
--
diff --git a/src/kudu/master/ts_descriptor.cc b/src/kudu/master/ts_descriptor.cc
index a360ccb..409cc85 100644
--- a/src/kudu/master/ts_descriptor.cc
+++ b/src/kudu/master/ts_descriptor.cc
@@ -100,7 +100,7 @@ bool IsValidLocation(const string& location) {
 Status GetLocationFromLocationMappingCmd(const string& cmd,
  const string& host,
  string* location) {
-  DCHECK_NOTNULL(location);
+  DCHECK(location);
   vector argv = strings::Split(cmd, " ", strings::SkipEmpty());
   if (argv.empty()) {
 return Status::RuntimeError("invalid empty location mapping command");

[3/3] kudu git commit: move KUDU_NO_EXPORT attribute on methods

[danburkert]

move KUDU_NO_EXPORT attribute on methods

The original location was ineffective due to a bug in older versions of
GCC, [1] appears to be the best tracking issue. Adding the annotation at
the end of the method signature silences the warnings on GCC 4.8, and
maintains symbol hiding as checked with nm.

[1]: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=30361

Change-Id: Id0ebdf97263065e14e7a43cf9fcfa7cb26953f57
Reviewed-on: http://gerrit.cloudera.org:8080/11415
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ed60c11a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ed60c11a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ed60c11a

Branch: refs/heads/master
Commit: ed60c11a3e2fa78e32edbb4f5d3cf844d23d8f00
Parents: d91a252
Author: Dan Burkert 
Authored: Mon Sep 10 16:36:13 2018 -0700
Committer: Dan Burkert 
Committed: Tue Sep 11 22:32:50 2018 +

--
 src/kudu/client/client.h | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ed60c11a/src/kudu/client/client.h
--
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index bfc2c2b..795aae5 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -347,8 +347,8 @@ class KUDU_EXPORT KuduClient : public 
sp::enable_shared_from_this {
   ///   Whether to apply the deletion to external catalogs, such as the Hive 
Metastore,
   ///   which the Kudu master has been configured to integrate with.
   /// @return Operation status.
-  Status KUDU_NO_EXPORT DeleteTableInCatalogs(const std::string& table_name,
-  bool modify_external_catalogs);
+  Status DeleteTableInCatalogs(const std::string& table_name,
+   bool modify_external_catalogs) KUDU_NO_EXPORT;
 
   /// Create a KuduTableAlterer object.
   ///
@@ -445,8 +445,8 @@ class KUDU_EXPORT KuduClient : public 
sp::enable_shared_from_this {
   /// @param [out] tablet
   ///   Tablet information. The caller takes ownership of the tablet.
   /// @return Status object for the operation.
-  Status KUDU_NO_EXPORT GetTablet(const std::string& tablet_id,
-  KuduTablet** tablet);
+  Status GetTablet(const std::string& tablet_id,
+   KuduTablet** tablet) KUDU_NO_EXPORT;
 
   /// @endcond
 
@@ -547,12 +547,12 @@ class KUDU_EXPORT KuduClient : public 
sp::enable_shared_from_this {
   // @return the configured Hive Metastore URIs on the most recently connected 
to
   //leader master, or an empty string if the Hive Metastore integration is 
not
   //enabled.
-  std::string KUDU_NO_EXPORT GetHiveMetastoreUris() const;
+  std::string GetHiveMetastoreUris() const KUDU_NO_EXPORT;
 
   // @return the configured Hive Metastore SASL (Kerberos) configuration on 
the most
   //recently connected to leader master, or an arbitrary value if the Hive
   //Metastore integration is not enabled.
-  bool KUDU_NO_EXPORT GetHiveMetastoreSaslEnabled() const;
+  bool GetHiveMetastoreSaslEnabled() const KUDU_NO_EXPORT;
 
   // @return a unique ID which identifies the Hive Metastore instance, if the
   //cluster is configured with the Hive Metastore integration, or an
@@ -561,7 +561,7 @@ class KUDU_EXPORT KuduClient : public 
sp::enable_shared_from_this {
   // @note this is provided on a best-effort basis, as not all Hive Metastore
   //versions which Kudu is compatible with include the necessary APIs. See
   //HIVE-16452 for more info.
-  std::string KUDU_NO_EXPORT GetHiveMetastoreUuid() const;
+  std::string GetHiveMetastoreUuid() const KUDU_NO_EXPORT;
 
  private:
   class KUDU_NO_EXPORT Data;
@@ -1214,7 +1214,7 @@ class KUDU_EXPORT KuduTableAlterer {
   /// @param [in] modify_external_catalogs
   ///   Whether to apply the alteration to external catalogs.
   /// @return Raw pointer to this alterer object.
-  KuduTableAlterer* KUDU_NO_EXPORT modify_external_catalogs(bool 
modify_external_catalogs);
+  KuduTableAlterer* modify_external_catalogs(bool modify_external_catalogs) 
KUDU_NO_EXPORT;
 
   /// @return Status of the ALTER TABLE operation. The return value
   ///   may indicate an error in the alter operation,

[2/3] kudu git commit: build: add option to ignore test failures

[danburkert]

build: add option to ignore test failures

Sometimes when running build-and-test.sh, we may not care about the
failure of any given test, so long as the tests ran, e.g. if running
only for the sake of reporting the results of the tests. As such, this
patch adds an option to ignore test failures during runs of
build-and-test.sh.

I tested this by injecting errors into a test and doing a
build-and-test.sh run with the new ERROR_ON_TEST_FAILURE=1, verifying
that it still returns an error; and with ERROR_ON_TEST_FAILURE=0,
verifying that it returns no error. I also broke the build and verified
that, despite ERROR_ON_TEST_FAILURE=0, it returns an error.

Change-Id: I27ecacd499423755dbf742bb70db6a7f1e5c9db7
Reviewed-on: http://gerrit.cloudera.org:8080/11399
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins
Reviewed-by: Grant Henke 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d91a2525
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d91a2525
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d91a2525

Branch: refs/heads/master
Commit: d91a2525060d52bc27661b2b5978a288a71bcec2
Parents: 7856ec9
Author: Andrew Wong 
Authored: Thu Sep 6 18:10:45 2018 -0700
Committer: Andrew Wong 
Committed: Tue Sep 11 21:24:22 2018 +

--
 build-support/jenkins/build-and-test.sh | 33 ++--
 1 file changed, 21 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d91a2525/build-support/jenkins/build-and-test.sh
--
diff --git a/build-support/jenkins/build-and-test.sh 
b/build-support/jenkins/build-and-test.sh
index d1a3e42..312ff65 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -82,6 +82,9 @@
 #
 #   GRADLE_FLAGS   Default: ""
 # Extra flags which are passed to 'gradle' when running Gradle commands.
+#
+#   ERROR_ON_TEST_FAILUREDefault: 1
+# Whether test failures will cause this script to return an error.
 
 # If a commit messages contains a line that says 'DONT_BUILD', exit
 # immediately.
@@ -121,6 +124,7 @@ BUILD_MAVEN=${BUILD_MAVEN:-0}
 BUILD_GRADLE=${BUILD_GRADLE:-1}
 BUILD_PYTHON=${BUILD_PYTHON:-1}
 BUILD_PYTHON3=${BUILD_PYTHON3:-1}
+ERROR_ON_TEST_FAILURE=${ERROR_ON_TEST_FAILURE:-1}
 
 # Ensure that the test data directory is usable.
 mkdir -p "$TEST_TMPDIR"
@@ -329,6 +333,7 @@ if [ "$RUN_FLAKY_ONLY" == "1" ] ; then
   BUILD_JAVA=0
 fi
 
+TESTS_FAILED=0
 EXIT_STATUS=0
 FAILURES=""
 
@@ -350,7 +355,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then
 fi
 
 if ! $THIRDPARTY_BIN/ctest -j$NUM_PROCS $EXTRA_TEST_FLAGS ; then
-  EXIT_STATUS=1
+  TESTS_FAILED=1
   FAILURES="$FAILURES"$'C++ tests failed\n'
 fi
 
@@ -388,7 +393,7 @@ if [ "$BUILD_JAVA" == "1" ]; then
 EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS -Dmaven.javadoc.skip"
 EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS $MVN_FLAGS"
 if ! mvn $EXTRA_MVN_FLAGS clean verify ; then
-  EXIT_STATUS=1
+  TESTS_FAILED=1
   FAILURES="$FAILURES"$'Java Maven build/test failed\n'
 fi
   fi
@@ -418,7 +423,7 @@ if [ "$BUILD_JAVA" == "1" ]; then
 else
   # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis 
is fixed
   if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then
-EXIT_STATUS=1
+TESTS_FAILED=1
 FAILURES="$FAILURES"$'Java Gradle build/test failed\n'
   fi
 fi
@@ -501,7 +506,7 @@ if [ "$BUILD_PYTHON" == "1" ]; then
   if ! python setup.py test \
   --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \
   2> $TEST_LOGDIR/python_client.log ; then
-EXIT_STATUS=1
+TESTS_FAILED=1
 FAILURES="$FAILURES"$'Python tests failed\n'
   fi
 
@@ -559,7 +564,7 @@ if [ "$BUILD_PYTHON3" == "1" ]; then
   if ! python setup.py test \
   --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \
   2> $TEST_LOGDIR/python3_client.log ; then
-EXIT_STATUS=1
+TESTS_FAILED=1
 FAILURES="$FAILURES"$'Python 3 tests failed\n'
   fi
 
@@ -574,7 +579,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then
   echo 
   C_DIST_TEST_ID=`cat $BUILD_ROOT/c-dist-test-job-id`
   if ! $DIST_TEST_HOME/bin/client watch $C_DIST_TEST_ID ; then
-EXIT_STATUS=1
+TESTS_FAILED=1
 FAILURES="$FAILURES"$'Distributed C++ tests failed\n'
   fi
   DT_DIR=$TEST_LOGDIR/dist-test-out
@@ -594,7 +599,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then
 echo 
 JAVA_DIST_TEST_ID=`cat $BUILD_ROOT/java-dist-test-job-id`
 if ! $DIST_TEST_HOME/bin/client watch $JAVA_DIST_TEST_ID ; then
-  EXIT_STATUS=1
+  TESTS_FAILED=1
   FAILURES="$FAILURES"$'Distributed Java tests failed\n'
 fi
 

[2/2] kudu git commit: KUDU-428: Sentry integration scaffolding

[danburkert]

KUDU-428: Sentry integration scaffolding

In preparation for KUDU-428 (Sentry integration), this commit introduces
some basic module and build-system scaffolding, including a new 'sentry'
module, as well as the Sentry service thrift definition files. This is
heavily based on the HMS integration equivalents.

Change-Id: I51e68299b97f74f6844bfa9f8aba4c0bd4246c11
Reviewed-on: http://gerrit.cloudera.org:8080/11292
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/846eb7f9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/846eb7f9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/846eb7f9

Branch: refs/heads/master
Commit: 846eb7f9705812515fadffe1eacb69bf60ad5a9b
Parents: 81ab99b
Author: Dan Burkert 
Authored: Fri Aug 17 12:05:53 2018 -0700
Committer: Dan Burkert 
Committed: Tue Aug 28 23:34:17 2018 +

--
 CMakeLists.txt   |   3 +-
 cmake_modules/FindThrift.cmake   |  14 +-
 src/kudu/hms/CMakeLists.txt  |  16 +-
 src/kudu/mini-cluster/CMakeLists.txt |   2 +-
 src/kudu/sentry/CMakeLists.txt   |  67 
 src/kudu/sentry/mini_sentry.cc   |  23 ++
 src/kudu/sentry/mini_sentry.h|  28 ++
 src/kudu/sentry/sentry_client-test.cc|  39 +++
 src/kudu/sentry/sentry_client.cc |  23 ++
 src/kudu/sentry/sentry_client.h  |  28 ++
 src/kudu/sentry/sentry_common_service.thrift |  50 +++
 src/kudu/sentry/sentry_policy_service.thrift | 370 ++
 src/kudu/sentry/thrift_operators.cc  |  62 
 13 files changed, 717 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/846eb7f9/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7029a4a..0cd6bc1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1374,7 +1374,7 @@ endif (UNIX)
 
 if (UNIX)
   add_custom_target(iwyu-generated-headers
-DEPENDS pb-gen krpc-gen hms_thrift)
+DEPENDS pb-gen krpc-gen hms_thrift sentry_thrift)
   add_custom_target(iwyu ${BUILD_SUPPORT_DIR}/iwyu.py --from-git
 DEPENDS iwyu-generated-headers)
   add_custom_target(iwyu-fix ${BUILD_SUPPORT_DIR}/iwyu.py --fix --from-git
@@ -1474,6 +1474,7 @@ add_subdirectory(src/kudu/master)
 add_subdirectory(src/kudu/mini-cluster)
 add_subdirectory(src/kudu/rpc)
 add_subdirectory(src/kudu/security)
+add_subdirectory(src/kudu/sentry)
 add_subdirectory(src/kudu/server)
 add_subdirectory(src/kudu/tablet)
 add_subdirectory(src/kudu/tools)

http://git-wip-us.apache.org/repos/asf/kudu/blob/846eb7f9/cmake_modules/FindThrift.cmake
--
diff --git a/cmake_modules/FindThrift.cmake b/cmake_modules/FindThrift.cmake
index 834bbfd..3b9b217 100644
--- a/cmake_modules/FindThrift.cmake
+++ b/cmake_modules/FindThrift.cmake
@@ -51,6 +51,8 @@
 #  custom targets; if SRCS/HDRS need to be used in multiple
 #  libraries, those libraries should depend on these targets
 #  in order to "serialize" the thrift invocations
+#   FB303 = Option which determines if the Thrift definitions depend on the
+#   FB303 support library.
 #  
 
 function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
@@ -59,7 +61,7 @@ function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
 return()
   endif(NOT ARGN)
 
-  set(options)
+  set(options FB303)
   set(one_value_args SOURCE_ROOT BINARY_ROOT)
   set(multi_value_args EXTRA_THRIFT_PATHS THRIFT_FILES)
   cmake_parse_arguments(ARG "${options}" "${one_value_args}" 
"${multi_value_args}" ${ARGN})
@@ -102,9 +104,13 @@ function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
   list(APPEND THRIFT_CC_OUT "${ARG_BINARY_ROOT}/${SERVICE}.cpp")
 endforeach()
 
-# TODO(dan): Add the fb303 files manually. This is a complete hack.
-list(APPEND ${SRCS} "${THRIFT_CC_OUT}" "fb303_types.cpp" 
"fb303_constants.cpp" "FacebookService.cpp")
-list(APPEND ${HDRS} "${THRIFT_H_OUT}" "fb303_types.h" "fb303_constants.h" 
"FacebookService.h")
+list(APPEND ${SRCS} "${THRIFT_CC_OUT}")
+list(APPEND ${HDRS} "${THRIFT_H_OUT}")
+
+if(ARG_FB303)
+  list(APPEND ${SRCS} fb303_types.cpp fb303_constants.cpp 
FacebookService.cpp)
+  list(APPEND ${HDRS} fb303_types.h fb303_constants.h FacebookService.h)
+endif()
 
 add_custom_command(
   OUTPUT ${THRIFT_CC_OUT} ${THRIFT_H_OUT}

http://git-wip-us.apache.org/repos/asf/kudu/blob/846eb7f9/src/kudu/hms/CMakeLists.txt
--
diff --git a/src/kudu/hms/CMakeLists.txt 

kudu git commit: Add unstable client APIs to fetch HMS integration configuration status

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master a8d088e11 -> 2c572cbd4


Add unstable client APIs to fetch HMS integration configuration status

The Impala/Kudu integration must know whether the Kudu cluster is
configured with the HMS integration enabled. This commit adds an
unstable API to the Java client and a private API to the Kudu client to
lookup this information. The new C++ API is utilized by the HMS tools to
get the HMS connection information without having to use the GetFlags
API, which requires admin privileges.

The patch required adding support for enabling the HMS integration to
the Java mini cluster.

Change-Id: Iddb89787ed35c41e85f1d9bf953c4c228dcafcdb
Reviewed-on: http://gerrit.cloudera.org:8080/11121
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2c572cbd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2c572cbd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2c572cbd

Branch: refs/heads/master
Commit: 2c572cbd4dbf4c35d5d85521cae6067722e346a0
Parents: a8d088e
Author: Dan Burkert 
Authored: Fri Aug 3 16:14:03 2018 -0700
Committer: Dan Burkert 
Committed: Fri Aug 24 22:25:00 2018 +

--
 .../org/apache/kudu/client/AsyncKuduClient.java | 50 ++-
 .../apache/kudu/client/HiveMetastoreConfig.java | 52 
 .../java/org/apache/kudu/client/KuduClient.java | 14 ++
 .../org/apache/kudu/client/MiniKuduCluster.java | 18 +--
 .../apache/kudu/client/TestMiniKuduCluster.java | 15 ++
 src/kudu/client/client-internal.cc  | 11 -
 src/kudu/client/client-internal.h   |  9 +++-
 src/kudu/client/client.cc   | 17 +++
 src/kudu/client/client.h| 19 +++
 src/kudu/hms/mini_hms.cc| 26 ++
 src/kudu/hms/mini_hms.h | 10 +++-
 src/kudu/master/master.proto| 27 ++
 src/kudu/master/master_service.cc   | 11 +
 src/kudu/mini-cluster/external_mini_cluster.cc  |  1 +
 src/kudu/tools/tool_action_hms.cc   | 34 +
 15 files changed, 271 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/2c572cbd/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
--
diff --git 
a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java 
b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index 8e97baf..8c1e032 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -294,6 +294,10 @@ public class AsyncKuduClient implements AutoCloseable {
   @GuardedBy("sessions")
   private final Set sessions = new HashSet<>();
 
+  /** The Hive Metastore configuration of the most recently connected-to 
master. */
+  @GuardedBy("this")
+  private HiveMetastoreConfig hiveMetastoreConfig = null;
+
   // Since RPCs to the masters also go through RpcProxy, we need to treat them 
as if they were a
   // normal table. We'll use the following fake table name to identify places 
where we need special
   // handling.
@@ -815,7 +819,9 @@ public class AsyncKuduClient implements AutoCloseable {
 // We have no authn data -- connect to the master, which will fetch
 // new info.
 return getMasterTableLocationsPB(null)
-.addCallback(new MasterLookupCB(masterTable, null, 1))
+.addCallback(new MasterLookupCB(masterTable,
+/* partitionKey */ null,
+/* requestedBatchSize */ 1))
 .addCallback(new Callback() {
   @Override
   public byte[] call(Object ignored) {
@@ -827,6 +833,35 @@ public class AsyncKuduClient implements AutoCloseable {
   }
 
   /**
+   * Get the Hive Metastore configuration of the most recently connected-to 
leader master, or
+   * {@code null} if the Hive Metastore integration is not enabled.
+   */
+  @InterfaceAudience.LimitedPrivate("Impala")
+  @InterfaceStability.Unstable
+  public Deferred getHiveMetastoreConfig() {
+// If we've already connected to the master, use the config we received 
when we connected.
+if (hasConnectedToMaster) {
+  synchronized (this) {
+return Deferred.fromResult(hiveMetastoreConfig);
+  }
+}
+// We have no Metastore config -- connect to the master, which will fetch 
new info.
+return getMasterTableLocationsPB(null)
+.addCallback(new MasterLookupCB(masterTable,
+/* partitionKey */ null,
+   

[2/6] kudu git commit: [webui] Add partition and on-disk size to /tablet

[danburkert]

[webui] Add partition and on-disk size to /tablet

Change-Id: If8de8e3232fc608dd31ab7275b29805b0abdc140
Reviewed-on: http://gerrit.cloudera.org:8080/11289
Reviewed-by: Alexey Serbin 
Tested-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ec419c64
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ec419c64
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ec419c64

Branch: refs/heads/master
Commit: ec419c64e74834082a894bb3e87975bc86478f6d
Parents: a6baa72
Author: Will Berkeley 
Authored: Tue Aug 21 13:03:49 2018 -0700
Committer: Will Berkeley 
Committed: Wed Aug 22 22:13:00 2018 +

--
 src/kudu/tserver/tserver_path_handlers.cc | 7 ++-
 www/tablet.mustache   | 5 +
 2 files changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ec419c64/src/kudu/tserver/tserver_path_handlers.cc
--
diff --git a/src/kudu/tserver/tserver_path_handlers.cc 
b/src/kudu/tserver/tserver_path_handlers.cc
index ddaa3c1..2f6c4b5 100644
--- a/src/kudu/tserver/tserver_path_handlers.cc
+++ b/src/kudu/tserver/tserver_path_handlers.cc
@@ -412,7 +412,12 @@ void TabletServerPathHandlers::HandleTabletPage(const 
Webserver::WebRequest& req
   output->Set("role", RaftPeerPB::Role_Name(role));
   output->Set("table_name", table_name);
 
-  const Schema& schema = replica->tablet_metadata()->schema();
+  const auto& tmeta = replica->tablet_metadata();
+  const Schema& schema = tmeta->schema();
+  output->Set("partition",
+  
tmeta->partition_schema().PartitionDebugString(tmeta->partition(), schema));
+  output->Set("on_disk_size", 
HumanReadableNumBytes::ToString(replica->OnDiskSize()));
+
   SchemaToJson(schema, output);
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/ec419c64/www/tablet.mustache
--
diff --git a/www/tablet.mustache b/www/tablet.mustache
index 8f80581..aa83558 100644
--- a/www/tablet.mustache
+++ b/www/tablet.mustache
@@ -23,6 +23,11 @@ under the License.
   Tablet {{tablet_id}} ({{state}}/{{role}})
   Table {{table_name}}
 
+  
+Partition{{partition}}
+On-disk Size{{on_disk_size}}
+  
+
   Schema
   
 

[5/6] kudu git commit: [dist-test] Fix the descriptions of Java tests

[danburkert]

[dist-test] Fix the descriptions of Java tests

Removes the extra trailing “.isolate” in the descriptions
of Java tests run on dist-test.

Change-Id: I6fd893e46b0cb2f3c23900130dc2c904d909a2ae
Reviewed-on: http://gerrit.cloudera.org:8080/11300
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a8e33356
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a8e33356
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a8e33356

Branch: refs/heads/master
Commit: a8e3335653aef465ff6c3a900c92d9788f1dc8a8
Parents: 6ae9ecb
Author: Grant Henke 
Authored: Thu Aug 23 11:19:34 2018 -0500
Committer: Grant Henke 
Committed: Thu Aug 23 18:45:48 2018 +

--
 .../main/groovy/org/apache/kudu/gradle/DistTestTask.java| 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a8e33356/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
--
diff --git 
a/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java 
b/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
index bebd604..3645017 100644
--- a/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
+++ b/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
@@ -105,16 +105,17 @@ public class DistTestTask extends DefaultTask {
 for (Test t : testTasks) {
   List testClassNames = collectTestNames(t);
   for (String c : testClassNames) {
-File isoFile = new File(outputDir, c + ".isolate");
+File isolateFile = new File(outputDir, c + ".isolate");
+File isolatedFile = new File(outputDir, c + ".isolated");
 File genJsonFile = new File(outputDir, c + ".gen.json");
 
-Files.write(genIsolate(outputDir.toPath(), t, c, baseDeps), isoFile, 
UTF_8);
+Files.write(genIsolate(outputDir.toPath(), t, c, baseDeps), 
isolateFile, UTF_8);
 
 // Write the gen.json
 GenJson gen = new GenJson();
 gen.args = ImmutableList.of(
-"-i", isoFile.toString(),
-"-s", isoFile.toString() + ".isolated");
+"-i", isolateFile.toString(),
+"-s", isolatedFile.toString());
 gen.dir = outputDir.toString();
 gen.name = c;
 Files.write(GSON.toJson(gen), genJsonFile, UTF_8);

[4/6] kudu git commit: [webui] Fancy tables for /maintenance-manager

[danburkert]

[webui] Fancy tables for /maintenance-manager

Change-Id: I725c13249b2cdc95fa674135c75b1414b2fc9047
Reviewed-on: http://gerrit.cloudera.org:8080/11291
Reviewed-by: Alexey Serbin 
Tested-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6ae9ecbe
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6ae9ecbe
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6ae9ecbe

Branch: refs/heads/master
Commit: 6ae9ecbe2595090c78e7afd271aae9d04dd4d0b5
Parents: 9a01205
Author: Will Berkeley 
Authored: Tue Aug 21 15:00:56 2018 -0700
Committer: Will Berkeley 
Committed: Wed Aug 22 22:29:12 2018 +

--
 www/maintenance-manager.mustache | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/6ae9ecbe/www/maintenance-manager.mustache
--
diff --git a/www/maintenance-manager.mustache b/www/maintenance-manager.mustache
index a906f52..269a7d2 100644
--- a/www/maintenance-manager.mustache
+++ b/www/maintenance-manager.mustache
@@ -20,7 +20,7 @@ under the License.
 Maintenance Manager state
 
 Running operations
-
+
   
 
   Name
@@ -38,7 +38,7 @@ under the License.
 
 
 Recent completed operations
-
+
   
 
   Name
@@ -58,7 +58,7 @@ under the License.
 
 
 Non-running operations
-
+
   
 
   Name

[1/6] kudu git commit: [webui] Convert /tablet and its offshoots to mustache

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 9ad90eebe -> 36187ef62


[webui] Convert /tablet and its offshoots to mustache

This patch converts the following pages to mustache templates:

* /tablet
* /tablet-consensus-status
* /tablet-rowsetlayout-svg
* /log-anchors

It does the conversion as simply as possible and makes no changes to
the content exposed, except it linkifies the tablet id in a couple of
cases.

Follow-ups will make improvements and add more information.

Change-Id: Iea14f5504b9665b9c349efc9b3b13fc3fe19d8b2
Reviewed-on: http://gerrit.cloudera.org:8080/11288
Reviewed-by: Alexey Serbin 
Tested-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a6baa721
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a6baa721
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a6baa721

Branch: refs/heads/master
Commit: a6baa721076f9e7a91d0f108ef5b4a443c731f7c
Parents: 9ad90ee
Author: Will Berkeley 
Authored: Tue Aug 21 10:39:40 2018 -0700
Committer: Will Berkeley 
Committed: Wed Aug 22 22:12:25 2018 +

--
 src/kudu/server/webui_util.cc |  41 ---
 src/kudu/tserver/tserver_path_handlers.cc | 141 ++---
 src/kudu/tserver/tserver_path_handlers.h  |   8 +-
 www/log-anchors.mustache  |  25 +
 www/tablet-consensus-status.mustache  |  24 +
 www/tablet-rowsetlayout-svg.mustache  |  25 +
 www/tablet.mustache   |  66 
 7 files changed, 201 insertions(+), 129 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a6baa721/src/kudu/server/webui_util.cc
--
diff --git a/src/kudu/server/webui_util.cc b/src/kudu/server/webui_util.cc
index cd73042..590080f 100644
--- a/src/kudu/server/webui_util.cc
+++ b/src/kudu/server/webui_util.cc
@@ -17,7 +17,6 @@
 
 #include "kudu/server/webui_util.h"
 
-#include 
 #include 
 
 #include "kudu/common/common.pb.h"
@@ -29,7 +28,6 @@
 #include "kudu/util/compression/compression.pb.h"
 #include "kudu/util/easy_json.h"
 #include "kudu/util/monotime.h"
-#include "kudu/util/url-coding.h"
 
 using std::string;
 using strings::Substitute;
@@ -55,45 +53,6 @@ void SchemaToJson(const Schema& schema, EasyJson* output) {
   }
 }
 
-void HtmlOutputSchemaTable(const Schema& schema,
-   std::ostringstream* output) {
-  *output << "\n";
-  *output << "  "
-  << "ColumnIDType"
-  << "EncodingCompression"
-  << "Read defaultWrite default"
-  << "\n";
-  *output << "";
-  for (int i = 0; i < schema.num_columns(); i++) {
-const ColumnSchema& col = schema.column(i);
-const string& html_escaped_col_name = EscapeForHtmlToString(col.name());
-const string& col_name = schema.is_key_column(i) ?
- Substitute("$0", 
html_escaped_col_name) :
- html_escaped_col_name;
-string read_default = "-";
-if (col.has_read_default()) {
-  read_default = col.Stringify(col.read_default_value());
-}
-string write_default = "-";
-if (col.has_write_default()) {
-  write_default = col.Stringify(col.write_default_value());
-}
-const ColumnStorageAttributes& attrs = col.attributes();
-const string& encoding = EncodingType_Name(attrs.encoding);
-const string& compression = CompressionType_Name(attrs.compression);
-*output << Substitute("$0$1$2$3"
-  "$4$5$6\n",
-  col_name,
-  schema.column_id(i),
-  col.TypeToString(),
-  EscapeForHtmlToString(encoding),
-  EscapeForHtmlToString(compression),
-  EscapeForHtmlToString(read_default),
-  EscapeForHtmlToString(write_default));
-  }
-  *output << "\n";
-}
-
 void TaskListToJson(const std::vector >& tasks, 
EasyJson* output) {
   EasyJson tasks_json = output->Set("tasks", EasyJson::kArray);
   for (const scoped_refptr& task : tasks) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/a6baa721/src/kudu/tserver/tserver_path_handlers.cc
--
diff --git a/src/kudu/tserver/tserver_path_handlers.cc 
b/src/kudu/tserver/tserver_path_handlers.cc
index b8071ed..ddaa3c1 100644
--- a/src/kudu/tserver/tserver_path_handlers.cc
+++ b/src/kudu/tserver/tserver_path_handlers.cc
@@ -101,6 +101,16 @@ bool CompareByMemberType(const RaftPeerPB& a, const 
RaftPeerPB& b) {
   return a.member_type() < b.member_type();
 }
 
+string TabletLink(const string& id) {
+  return Substitute("$1",
+UrlEncodeToString(id),
+

[3/6] kudu git commit: [webui] Make the tables on /tablets fancy

[danburkert]

[webui] Make the tables on /tablets fancy

The /tablets page has live tablets and a tombstoned tablets tables.
These tables get big and unwieldy to use. This patch makes the tables
paginated and searchable by adding the bootstrap-table plugin[1]. I
picked this plugin because it integrates with our existing bootstrap
styles, it worked out of the box with no effort, and the project looks
active. I would've liked a plugin that could be used to sort columns
based on their type out of the box, but I couldn't find a good
candidate.

I also removed a couple of columns that I've never seen be useful:

* Write buffer memory usage: Users are not usually interested in this for
  specific tablets, except possibly to identify outliers, which can be
  done on the mem-trackers page.
* Last status: This is updated at tablet creation or bootstrap time and
  never contains useful information.

Less columns allows the table to fit comfortably on smaller screens.

1: https://github.com/wenzhixin/bootstrap-table

Change-Id: I5243e9cb839e75936c71b14fed08881317ee0f5f
Reviewed-on: http://gerrit.cloudera.org:8080/11290
Reviewed-by: Alexey Serbin 
Tested-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9a01205c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9a01205c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9a01205c

Branch: refs/heads/master
Commit: 9a01205ce10b468698c76aedbf51016a90045620
Parents: ec419c6
Author: Will Berkeley 
Authored: Tue Aug 21 13:49:26 2018 -0700
Committer: Will Berkeley 
Committed: Wed Aug 22 22:29:10 2018 +

--
 build-support/release/rat_exclude_files.txt |   2 +
 src/kudu/server/webserver.cc|   2 +
 src/kudu/tserver/tserver_path_handlers.cc   |   4 -
 www/bootstrap/css/bootstrap-table.min.css   |   7 ++
 www/bootstrap/js/bootstrap-table.min.js |   9 ++
 www/tablets.mustache| 105 ++-
 6 files changed, 67 insertions(+), 62 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9a01205c/build-support/release/rat_exclude_files.txt
--
diff --git a/build-support/release/rat_exclude_files.txt 
b/build-support/release/rat_exclude_files.txt
index ec24bdb..8680003 100644
--- a/build-support/release/rat_exclude_files.txt
+++ b/build-support/release/rat_exclude_files.txt
@@ -172,7 +172,9 @@ src/kudu/util/debug/trace_event_impl_constants.cc
 src/kudu/util/debug/trace_event_synthetic_delay.cc
 src/kudu/util/debug/trace_event_synthetic_delay.h
 thirdparty/patches/*.patch
+www/bootstrap/css/bootstrap-table.min.css
 www/bootstrap/css/bootstrap.min.css
+www/bootstrap/js/bootstrap-table.min.js
 www/bootstrap/js/bootstrap.min.js
 www/d3.v2.js
 www/epoch.0.5.2.min.css

http://git-wip-us.apache.org/repos/asf/kudu/blob/9a01205c/src/kudu/server/webserver.cc
--
diff --git a/src/kudu/server/webserver.cc b/src/kudu/server/webserver.cc
index 516cd89..0fc0aec 100644
--- a/src/kudu/server/webserver.cc
+++ b/src/kudu/server/webserver.cc
@@ -562,8 +562,10 @@ static const char* const kMainTemplate = R"(
 Kudu
 
 
+
 
 
+
 
   
   

http://git-wip-us.apache.org/repos/asf/kudu/blob/9a01205c/src/kudu/tserver/tserver_path_handlers.cc
--
diff --git a/src/kudu/tserver/tserver_path_handlers.cc 
b/src/kudu/tserver/tserver_path_handlers.cc
index 2f6c4b5..8fa0196 100644
--- a/src/kudu/tserver/tserver_path_handlers.cc
+++ b/src/kudu/tserver/tserver_path_handlers.cc
@@ -61,7 +61,6 @@
 #include "kudu/util/easy_json.h"
 #include "kudu/util/maintenance_manager.h"
 #include "kudu/util/maintenance_manager.pb.h"
-#include "kudu/util/mem_tracker.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/pb_util.h"
 #include "kudu/util/url-coding.h"
@@ -346,8 +345,6 @@ void TabletServerPathHandlers::HandleTabletsPage(const 
Webserver::WebRequest& /*
   replica_json["table_name"] = status.table_name();
   if (tablet != nullptr) {
 replica_json["id_or_link"] = TabletLink(status.tablet_id());
-replica_json["mem_bytes"] = HumanReadableNumBytes::ToString(
-tablet->mem_tracker()->consumption());
   } else {
 replica_json["id_or_link"] = status.tablet_id();
   }
@@ -369,7 +366,6 @@ void TabletServerPathHandlers::HandleTabletsPage(const 
Webserver::WebRequest& /*
 
local_uuid);
 }
   }
-  replica_json["last_status"] = status.last_status();
 }
   };
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/9a01205c/www/bootstrap/css/bootstrap-table.min.css

[6/6] kudu git commit: Enable iwyu on hms files

[danburkert]

Enable iwyu on hms files

Change-Id: I7d4a5b23683b4fbec12fa86c24d997677ee3113a
Reviewed-on: http://gerrit.cloudera.org:8080/11301
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/36187ef6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/36187ef6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/36187ef6

Branch: refs/heads/master
Commit: 36187ef62e9fb892ea286af8e9547790a3fd34ba
Parents: a8e3335
Author: Dan Burkert 
Authored: Wed Aug 22 15:08:14 2018 -0700
Committer: Dan Burkert 
Committed: Thu Aug 23 22:50:44 2018 +

--
 build-support/iwyu.py   | 6 --
 src/kudu/hms/hms_catalog.cc | 2 ++
 src/kudu/hms/hms_catalog.h  | 9 ++---
 src/kudu/hms/hms_client.h   | 9 +
 4 files changed, 5 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/36187ef6/build-support/iwyu.py
--
diff --git a/build-support/iwyu.py b/build-support/iwyu.py
index f267d0c..d665421 100755
--- a/build-support/iwyu.py
+++ b/build-support/iwyu.py
@@ -61,12 +61,6 @@ _MUTED_FILES = set([
   "src/kudu/common/encoded_key-test.cc",
   "src/kudu/common/schema.h",
   "src/kudu/experiments/rwlock-perf.cc",
-  "src/kudu/hms/hms_catalog.cc",
-  "src/kudu/hms/hms_catalog.h",
-  "src/kudu/hms/hms_client-test.cc",
-  "src/kudu/hms/hms_client.cc",
-  "src/kudu/hms/hms_client.h",
-  "src/kudu/hms/mini_hms.h",
   "src/kudu/master/catalog_manager.cc",
   "src/kudu/master/catalog_manager.h",
   "src/kudu/rpc/reactor.cc",

http://git-wip-us.apache.org/repos/asf/kudu/blob/36187ef6/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index ad62513..1b57ccb 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -20,6 +20,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -34,6 +35,7 @@
 #include "kudu/common/schema.h"
 #include "kudu/common/types.h"
 #include "kudu/gutil/macros.h"
+#include "kudu/gutil/map-util.h"
 #include "kudu/gutil/strings/ascii_ctype.h"
 #include "kudu/gutil/strings/charset.h"
 #include "kudu/gutil/strings/split.h"

http://git-wip-us.apache.org/repos/asf/kudu/blob/36187ef6/src/kudu/hms/hms_catalog.h
--
diff --git a/src/kudu/hms/hms_catalog.h b/src/kudu/hms/hms_catalog.h
index ebb1f41..bb807a1 100644
--- a/src/kudu/hms/hms_catalog.h
+++ b/src/kudu/hms/hms_catalog.h
@@ -25,21 +25,16 @@
 
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/port.h"
+#include "kudu/hms/hive_metastore_types.h"
 #include "kudu/hms/hms_client.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/net/net_util.h"
 #include "kudu/util/status.h"
 
-class StringPiece;
-
-namespace hive {
-class NotificationEvent;
-class Table;
-}
-
 namespace kudu {
 
 class Schema;
+class Slice;
 class ThreadPool;
 
 namespace hms {

http://git-wip-us.apache.org/repos/asf/kudu/blob/36187ef6/src/kudu/hms/hms_client.h
--
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index 82ac324..6c97d1c 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -23,18 +23,11 @@
 
 #include "kudu/gutil/port.h"
 #include "kudu/hms/ThriftHiveMetastore.h"
+#include "kudu/hms/hive_metastore_types.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 
-namespace hive {
-class Database;
-class EnvironmentContext;
-class NotificationEvent;
-class Partition;
-class Table;
-}
-
 namespace kudu {
 
 class HostPort;

kudu git commit: Bump Hive version, normalize hive package name

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 368990140 -> 60020cf32


Bump Hive version, normalize hive package name

This commit bumps the Hive version in thirdparty, and strips the
'apache' prefix and 'bin' suffix from the thirdparty tarball package,
which brings it inline with the hadoop package naming.

Change-Id: I5cf6696e2a49986aba025f13653dc549e6fc5beb
Reviewed-on: http://gerrit.cloudera.org:8080/11277
Reviewed-by: Adar Dembo 
Reviewed-by: Hao Hao 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/60020cf3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/60020cf3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/60020cf3

Branch: refs/heads/master
Commit: 60020cf329aed49624a9a3e4001c3fe4f55816f5
Parents: 3689901
Author: Dan Burkert 
Authored: Mon Aug 20 14:27:45 2018 -0700
Committer: Dan Burkert 
Committed: Tue Aug 21 20:19:00 2018 +

--
 build-support/run_dist_test.py | 2 +-
 thirdparty/package-hive.sh | 7 +--
 thirdparty/vars.sh | 4 ++--
 3 files changed, 8 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/60020cf3/build-support/run_dist_test.py
--
diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py
index 5c98299..624ec80 100755
--- a/build-support/run_dist_test.py
+++ b/build-support/run_dist_test.py
@@ -138,7 +138,7 @@ def main():
 
   # Add environment variables for Java dependencies. These environment 
variables
   # are used in mini_hms.cc.
-  env['HIVE_HOME'] = glob.glob(os.path.join(ROOT, 
"thirdparty/src/apache-hive-*-bin"))[0]
+  env['HIVE_HOME'] = glob.glob(os.path.join(ROOT, "thirdparty/src/hive-*"))[0]
   env['HADOOP_HOME'] = glob.glob(os.path.join(ROOT, 
"thirdparty/src/hadoop-*"))[0]
   env['JAVA_HOME'] = glob.glob("/usr/lib/jvm/java-1.8.0-*")[0]
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/60020cf3/thirdparty/package-hive.sh
--
diff --git a/thirdparty/package-hive.sh b/thirdparty/package-hive.sh
index d528806..5eedde5 100755
--- a/thirdparty/package-hive.sh
+++ b/thirdparty/package-hive.sh
@@ -34,7 +34,7 @@ set -eux
 
 ARTIFACT=apache-hive-$VERSION-bin
 
-wget https://archive.apache.org/dist/hive/hive-$VERSION/$ARTIFACT.tar.gz
+curl --retry 3 -L -O 
https://archive.apache.org/dist/hive/hive-$VERSION/$ARTIFACT.tar.gz
 tar xf $ARTIFACT.tar.gz
 
 for PROJECT in accumulo aether avatica calcite curator druid groovy hbase 
icu4j jetty jsp maven parquet zookeeper; do
@@ -47,4 +47,7 @@ rm -rf $ARTIFACT/lib/php
 rm -rf $ARTIFACT/lib/python
 rm -rf $ARTIFACT/lib/py
 
-tar czf $ARTIFACT-stripped.tar.gz $ARTIFACT
+# Remove the 'apache-' prefix and '-bin' suffix in order to normalize the
+# tarball with Hadoop and the rest of our thirdparty dependencies.
+mv $ARTIFACT hive-$VERSION
+tar czf hive-$VERSION-stripped.tar.gz hive-$VERSION

http://git-wip-us.apache.org/repos/asf/kudu/blob/60020cf3/thirdparty/vars.sh
--
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 0483906..2804bb3 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -212,8 +212,8 @@ BISON_SOURCE=$TP_SOURCE_DIR/$BISON_NAME
 # are published. The SHA below is the current head of branch-2.
 # Note: The Hive release binary tarball is stripped of unnecessary jars before
 # being uploaded. See thirdparty/package-hive.sh for details.
-HIVE_VERSION=c0a57a33b44d73fbe0f95256f0a3822bd3720aa8
-HIVE_NAME=apache-hive-$HIVE_VERSION-bin
+HIVE_VERSION=498021fa15186aee8b282d3c032fbd2cede6bec4
+HIVE_NAME=hive-$HIVE_VERSION
 HIVE_SOURCE=$TP_SOURCE_DIR/$HIVE_NAME
 
 # Note: The Hadoop release tarball is stripped of unnecessary jars before being

kudu git commit: KUDU-2540: Authorization failures on exactly-once RPCs cause FATAL

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.7.x cd28c7553 -> b24f166e3


KUDU-2540: Authorization failures on exactly-once RPCs cause FATAL

See the associated JIRA for bug details. The fix follows a slightly
different approach than suggested by Todd, namely initialization of the
RPC tracker in the RPC context is delayed until after authorization.
This allows the error path code to remain unchanged, and allows the one
caller of the RpcContext constructor to be slightly cleaned up.

Change-Id: I9717d36e438bbe68172fa2619c9829ad5fdc779d
Reviewed-on: http://gerrit.cloudera.org:8080/11216
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins
Reviewed-on: http://gerrit.cloudera.org:8080/11219
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b24f166e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b24f166e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b24f166e

Branch: refs/heads/branch-1.7.x
Commit: b24f166e356d7f34f6cff2efa0dcb3ff3ebe6e0b
Parents: cd28c75
Author: Dan Burkert 
Authored: Tue Aug 14 12:18:32 2018 -0700
Committer: Dan Burkert 
Committed: Tue Aug 14 22:55:34 2018 +

--
 src/kudu/rpc/rpc_context.cc   | 11 +++
 src/kudu/rpc/rpc_context.h| 10 --
 src/kudu/rpc/rpc_stub-test.cc | 36 +---
 src/kudu/rpc/service_if.cc| 12 +++-
 4 files changed, 47 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/b24f166e/src/kudu/rpc/rpc_context.cc
--
diff --git a/src/kudu/rpc/rpc_context.cc b/src/kudu/rpc/rpc_context.cc
index 123d21f..fe6fb38 100644
--- a/src/kudu/rpc/rpc_context.cc
+++ b/src/kudu/rpc/rpc_context.cc
@@ -48,12 +48,10 @@ namespace rpc {
 
 RpcContext::RpcContext(InboundCall *call,
const google::protobuf::Message *request_pb,
-   google::protobuf::Message *response_pb,
-   const scoped_refptr& result_tracker)
+   google::protobuf::Message *response_pb)
   : call_(CHECK_NOTNULL(call)),
 request_pb_(request_pb),
-response_pb_(response_pb),
-result_tracker_(result_tracker) {
+response_pb_(response_pb) {
   VLOG(4) << call_->remote_method().service_name() << ": Received RPC request 
for "
   << call_->ToString() << ":" << std::endl << 
SecureDebugString(*request_pb_);
   TRACE_EVENT_ASYNC_BEGIN2("rpc_call", "RPC", this,
@@ -64,6 +62,11 @@ RpcContext::RpcContext(InboundCall *call,
 RpcContext::~RpcContext() {
 }
 
+void RpcContext::SetResultTracker(scoped_refptr result_tracker) 
{
+  DCHECK(!result_tracker_);
+  result_tracker_ = std::move(result_tracker);
+}
+
 void RpcContext::RespondSuccess() {
   if (AreResultsTracked()) {
 result_tracker_->RecordCompletionAndRespond(call_->header().request_id(),

http://git-wip-us.apache.org/repos/asf/kudu/blob/b24f166e/src/kudu/rpc/rpc_context.h
--
diff --git a/src/kudu/rpc/rpc_context.h b/src/kudu/rpc/rpc_context.h
index a34c1a1..e27 100644
--- a/src/kudu/rpc/rpc_context.h
+++ b/src/kudu/rpc/rpc_context.h
@@ -69,11 +69,17 @@ class RpcContext {
   // and is not a public API.
   RpcContext(InboundCall *call,
  const google::protobuf::Message *request_pb,
- google::protobuf::Message *response_pb,
- const scoped_refptr& result_tracker);
+ google::protobuf::Message *response_pb);
 
   ~RpcContext();
 
+  // Initialize a result tracker for the RPC.
+  //
+  // This is delayed until after the constructor in order to allow for RPCs to
+  // be validated and used prior to initializing the tracking (primarily for
+  // authorization).
+  void SetResultTracker(scoped_refptr result_tracker);
+
   // Return the trace buffer for this call.
   Trace* trace();
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/b24f166e/src/kudu/rpc/rpc_stub-test.cc
--
diff --git a/src/kudu/rpc/rpc_stub-test.cc b/src/kudu/rpc/rpc_stub-test.cc
index d5b85d1..7e46dc9 100644
--- a/src/kudu/rpc/rpc_stub-test.cc
+++ b/src/kudu/rpc/rpc_stub-test.cc
@@ -208,13 +208,35 @@ TEST_F(RpcStubTest, TestAuthorization) {
 p.set_user_credentials(creds);
 
 // Alice is disallowed by all RPCs.
-RpcController controller;
-WhoAmIRequestPB req;
-WhoAmIResponsePB resp;
-Status s = p.WhoAmI(req, , );
-ASSERT_FALSE(s.ok());
-ASSERT_EQ(s.ToString(),
-  "Remote error: Not authorized: alice is not allowed to call this 
method");
+{
+  RpcController controller;
+  WhoAmIRequestPB req;
+  WhoAmIResponsePB resp;
+  Status s = p.WhoAmI(req, , );
+  

kudu git commit: KUDU-2540: Authorization failures on exactly-once RPCs cause FATAL

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.6.x 6dfc10ac0 -> c50b7fe69


KUDU-2540: Authorization failures on exactly-once RPCs cause FATAL

See the associated JIRA for bug details. The fix follows a slightly
different approach than suggested by Todd, namely initialization of the
RPC tracker in the RPC context is delayed until after authorization.
This allows the error path code to remain unchanged, and allows the one
caller of the RpcContext constructor to be slightly cleaned up.

Change-Id: I9717d36e438bbe68172fa2619c9829ad5fdc779d
Reviewed-on: http://gerrit.cloudera.org:8080/11216
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins
Reviewed-on: http://gerrit.cloudera.org:8080/11220
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c50b7fe6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c50b7fe6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c50b7fe6

Branch: refs/heads/branch-1.6.x
Commit: c50b7fe6915febcb2af480e67a1ca43a708142c0
Parents: 6dfc10a
Author: Dan Burkert 
Authored: Tue Aug 14 12:18:32 2018 -0700
Committer: Dan Burkert 
Committed: Tue Aug 14 22:55:37 2018 +

--
 src/kudu/rpc/rpc_context.cc   | 11 +++
 src/kudu/rpc/rpc_context.h| 10 --
 src/kudu/rpc/rpc_stub-test.cc | 36 +---
 src/kudu/rpc/service_if.cc| 12 +++-
 4 files changed, 47 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c50b7fe6/src/kudu/rpc/rpc_context.cc
--
diff --git a/src/kudu/rpc/rpc_context.cc b/src/kudu/rpc/rpc_context.cc
index 9c3d154..27011b5 100644
--- a/src/kudu/rpc/rpc_context.cc
+++ b/src/kudu/rpc/rpc_context.cc
@@ -47,12 +47,10 @@ namespace rpc {
 
 RpcContext::RpcContext(InboundCall *call,
const google::protobuf::Message *request_pb,
-   google::protobuf::Message *response_pb,
-   const scoped_refptr& result_tracker)
+   google::protobuf::Message *response_pb)
   : call_(CHECK_NOTNULL(call)),
 request_pb_(request_pb),
-response_pb_(response_pb),
-result_tracker_(result_tracker) {
+response_pb_(response_pb) {
   VLOG(4) << call_->remote_method().service_name() << ": Received RPC request 
for "
   << call_->ToString() << ":" << std::endl << 
SecureDebugString(*request_pb_);
   TRACE_EVENT_ASYNC_BEGIN2("rpc_call", "RPC", this,
@@ -63,6 +61,11 @@ RpcContext::RpcContext(InboundCall *call,
 RpcContext::~RpcContext() {
 }
 
+void RpcContext::SetResultTracker(scoped_refptr result_tracker) 
{
+  DCHECK(!result_tracker_);
+  result_tracker_ = std::move(result_tracker);
+}
+
 void RpcContext::RespondSuccess() {
   if (AreResultsTracked()) {
 result_tracker_->RecordCompletionAndRespond(call_->header().request_id(),

http://git-wip-us.apache.org/repos/asf/kudu/blob/c50b7fe6/src/kudu/rpc/rpc_context.h
--
diff --git a/src/kudu/rpc/rpc_context.h b/src/kudu/rpc/rpc_context.h
index 42f096e..468e6d1 100644
--- a/src/kudu/rpc/rpc_context.h
+++ b/src/kudu/rpc/rpc_context.h
@@ -68,11 +68,17 @@ class RpcContext {
   // and is not a public API.
   RpcContext(InboundCall *call,
  const google::protobuf::Message *request_pb,
- google::protobuf::Message *response_pb,
- const scoped_refptr& result_tracker);
+ google::protobuf::Message *response_pb);
 
   ~RpcContext();
 
+  // Initialize a result tracker for the RPC.
+  //
+  // This is delayed until after the constructor in order to allow for RPCs to
+  // be validated and used prior to initializing the tracking (primarily for
+  // authorization).
+  void SetResultTracker(scoped_refptr result_tracker);
+
   // Return the trace buffer for this call.
   Trace* trace();
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/c50b7fe6/src/kudu/rpc/rpc_stub-test.cc
--
diff --git a/src/kudu/rpc/rpc_stub-test.cc b/src/kudu/rpc/rpc_stub-test.cc
index 5fe2885..cb16c02 100644
--- a/src/kudu/rpc/rpc_stub-test.cc
+++ b/src/kudu/rpc/rpc_stub-test.cc
@@ -209,13 +209,35 @@ TEST_F(RpcStubTest, TestAuthorization) {
 p.set_user_credentials(creds);
 
 // Alice is disallowed by all RPCs.
-RpcController controller;
-WhoAmIRequestPB req;
-WhoAmIResponsePB resp;
-Status s = p.WhoAmI(req, , );
-ASSERT_FALSE(s.ok());
-ASSERT_EQ(s.ToString(),
-  "Remote error: Not authorized: alice is not allowed to call this 
method");
+{
+  RpcController controller;
+  WhoAmIRequestPB req;
+  WhoAmIResponsePB resp;
+  Status s = p.WhoAmI(req, , );
+  

kudu git commit: KUDU-2540: Authorization failures on exactly-once RPCs cause FATAL

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 2d92370d9 -> e795d5a82


KUDU-2540: Authorization failures on exactly-once RPCs cause FATAL

See the associated JIRA for bug details. The fix follows a slightly
different approach than suggested by Todd, namely initialization of the
RPC tracker in the RPC context is delayed until after authorization.
This allows the error path code to remain unchanged, and allows the one
caller of the RpcContext constructor to be slightly cleaned up.

Change-Id: I9717d36e438bbe68172fa2619c9829ad5fdc779d
Reviewed-on: http://gerrit.cloudera.org:8080/11216
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e795d5a8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e795d5a8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e795d5a8

Branch: refs/heads/master
Commit: e795d5a82c0c27c8a2ba1f82ac6695402aa42885
Parents: 2d92370
Author: Dan Burkert 
Authored: Tue Aug 14 12:18:32 2018 -0700
Committer: Dan Burkert 
Committed: Tue Aug 14 21:20:30 2018 +

--
 src/kudu/rpc/rpc_context.cc   | 11 +++
 src/kudu/rpc/rpc_context.h| 10 --
 src/kudu/rpc/rpc_stub-test.cc | 36 +---
 src/kudu/rpc/service_if.cc| 12 +++-
 4 files changed, 47 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/e795d5a8/src/kudu/rpc/rpc_context.cc
--
diff --git a/src/kudu/rpc/rpc_context.cc b/src/kudu/rpc/rpc_context.cc
index b3072ca..9f95358 100644
--- a/src/kudu/rpc/rpc_context.cc
+++ b/src/kudu/rpc/rpc_context.cc
@@ -48,12 +48,10 @@ namespace rpc {
 
 RpcContext::RpcContext(InboundCall *call,
const google::protobuf::Message *request_pb,
-   google::protobuf::Message *response_pb,
-   scoped_refptr result_tracker)
+   google::protobuf::Message *response_pb)
   : call_(CHECK_NOTNULL(call)),
 request_pb_(request_pb),
-response_pb_(response_pb),
-result_tracker_(std::move(result_tracker)) {
+response_pb_(response_pb) {
   VLOG(4) << call_->remote_method().service_name() << ": Received RPC request 
for "
   << call_->ToString() << ":" << std::endl << 
SecureDebugString(*request_pb_);
   TRACE_EVENT_ASYNC_BEGIN2("rpc_call", "RPC", this,
@@ -64,6 +62,11 @@ RpcContext::RpcContext(InboundCall *call,
 RpcContext::~RpcContext() {
 }
 
+void RpcContext::SetResultTracker(scoped_refptr result_tracker) 
{
+  DCHECK(!result_tracker_);
+  result_tracker_ = std::move(result_tracker);
+}
+
 void RpcContext::RespondSuccess() {
   if (AreResultsTracked()) {
 result_tracker_->RecordCompletionAndRespond(call_->header().request_id(),

http://git-wip-us.apache.org/repos/asf/kudu/blob/e795d5a8/src/kudu/rpc/rpc_context.h
--
diff --git a/src/kudu/rpc/rpc_context.h b/src/kudu/rpc/rpc_context.h
index 1b12023..38ce703 100644
--- a/src/kudu/rpc/rpc_context.h
+++ b/src/kudu/rpc/rpc_context.h
@@ -69,11 +69,17 @@ class RpcContext {
   // and is not a public API.
   RpcContext(InboundCall *call,
  const google::protobuf::Message *request_pb,
- google::protobuf::Message *response_pb,
- scoped_refptr result_tracker);
+ google::protobuf::Message *response_pb);
 
   ~RpcContext();
 
+  // Initialize a result tracker for the RPC.
+  //
+  // This is delayed until after the constructor in order to allow for RPCs to
+  // be validated and used prior to initializing the tracking (primarily for
+  // authorization).
+  void SetResultTracker(scoped_refptr result_tracker);
+
   // Return the trace buffer for this call.
   Trace* trace();
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/e795d5a8/src/kudu/rpc/rpc_stub-test.cc
--
diff --git a/src/kudu/rpc/rpc_stub-test.cc b/src/kudu/rpc/rpc_stub-test.cc
index e626276..dc92f2e 100644
--- a/src/kudu/rpc/rpc_stub-test.cc
+++ b/src/kudu/rpc/rpc_stub-test.cc
@@ -209,13 +209,35 @@ TEST_F(RpcStubTest, TestAuthorization) {
 p.set_user_credentials(creds);
 
 // Alice is disallowed by all RPCs.
-RpcController controller;
-WhoAmIRequestPB req;
-WhoAmIResponsePB resp;
-Status s = p.WhoAmI(req, , );
-ASSERT_FALSE(s.ok());
-ASSERT_EQ(s.ToString(),
-  "Remote error: Not authorized: alice is not allowed to call this 
method");
+{
+  RpcController controller;
+  WhoAmIRequestPB req;
+  WhoAmIResponsePB resp;
+  Status s = p.WhoAmI(req, , );
+  ASSERT_FALSE(s.ok());
+  ASSERT_EQ(s.ToString(),
+"Remote error: Not 

[1/4] kudu git commit: [tools] --report_only option for 'kudu cluster rebalance'

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 6c68e64d8 -> b2c642890


[tools] --report_only option for 'kudu cluster rebalance'

Added --report_only option for the 'kudu cluster rebalance' CLI tool
along with corresponding test.

Change-Id: Id8a81448a31ba76bb651b13e052f2c508cd0acd6
Reviewed-on: http://gerrit.cloudera.org:8080/11097
Reviewed-by: Will Berkeley 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/85a1808c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/85a1808c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/85a1808c

Branch: refs/heads/master
Commit: 85a1808c530f4cf857b4fe7fc5be2f7c00c6ed82
Parents: 6c68e64
Author: Alexey Serbin 
Authored: Tue Jul 31 16:55:13 2018 -0700
Committer: Alexey Serbin 
Committed: Wed Aug 1 19:33:53 2018 +

--
 src/kudu/tools/kudu-admin-test.cc | 38 ++
 src/kudu/tools/tool_action_cluster.cc |  9 +++
 2 files changed, 47 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/85a1808c/src/kudu/tools/kudu-admin-test.cc
--
diff --git a/src/kudu/tools/kudu-admin-test.cc 
b/src/kudu/tools/kudu-admin-test.cc
index 0408e35..04c5af9 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -1339,6 +1339,44 @@ static string ToolRunInfo(const Status& s, const string& 
out, const string& err)
   return str.str();
 }
 
+TEST_F(AdminCliTest, RebalancerReportOnly) {
+  static const char kReferenceOutput[] =
+R"***(Per-server replica distribution summary:
+   Statistic   |  Value
+---+--
+ Minimum Replica Count | 0
+ Maximum Replica Count | 1
+ Average Replica Count | 0.60
+
+Per-table replica distribution summary:
+ Replica Skew |  Value
+--+--
+ Minimum  | 1
+ Maximum  | 1
+ Average  | 1.00)***";
+
+  FLAGS_num_tablet_servers = 5;
+  NO_FATALS(BuildAndStart());
+
+  string out;
+  string err;
+  Status s = RunKuduTool({
+"cluster",
+"rebalance",
+cluster_->master()->bound_rpc_addr().ToString(),
+"--report_only",
+  }, , );
+  ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+  // The rebalancer should report on tablet replica distribution. The output
+  // should match the reference report: the distribution of the replicas
+  // is 100% repeatable given the number of tables created by the test,
+  // the replication factor and the number of tablet servers.
+  ASSERT_STR_CONTAINS(out, kReferenceOutput);
+  // The actual rebalancing should not run.
+  ASSERT_STR_NOT_CONTAINS(out, "rebalancing is complete:")
+  << ToolRunInfo(s, out, err);
+}
+
 // Make sure the rebalancer doesn't start if a tablet server is down.
 class RebalanceStartCriteriaTest :
 public AdminCliTest,

http://git-wip-us.apache.org/repos/asf/kudu/blob/85a1808c/src/kudu/tools/tool_action_cluster.cc
--
diff --git a/src/kudu/tools/tool_action_cluster.cc 
b/src/kudu/tools/tool_action_cluster.cc
index e420518..8fccd84 100644
--- a/src/kudu/tools/tool_action_cluster.cc
+++ b/src/kudu/tools/tool_action_cluster.cc
@@ -99,6 +99,10 @@ DEFINE_bool(output_replica_distribution_details, false,
 "Whether to output details on per-table and per-server "
 "replica distribution");
 
+DEFINE_bool(report_only, false,
+"Whether to report on table- and cluster-wide replica distribution 
"
+"skew and exit without doing any actual rebalancing");
+
 static bool ValidateMoveSingleReplicas(const char* flag_name,
const string& flag_value) {
   const vector allowed_values = { "auto", "enabled", "disabled" };
@@ -263,6 +267,10 @@ Status RunRebalance(const RunnerContext& context) {
   // Print info on pre-rebalance distribution of replicas.
   RETURN_NOT_OK(rebalancer.PrintStats(cout));
 
+  if (FLAGS_report_only) {
+return Status::OK();
+  }
+
   Rebalancer::RunStatus result_status;
   size_t moves_count;
   RETURN_NOT_OK(rebalancer.Run(_status, _count));
@@ -345,6 +353,7 @@ unique_ptr BuildClusterMode() {
 .AddOptionalParameter("max_staleness_interval_sec")
 .AddOptionalParameter("move_single_replicas")
 .AddOptionalParameter("output_replica_distribution_details")
+.AddOptionalParameter("report_only")
 .AddOptionalParameter("tables")
 .Build();
 builder.AddAction(std::move(rebalance));

[2/4] kudu git commit: hms tools: do not require HMS configuration flags

[danburkert]

hms tools: do not require HMS configuration flags

The HMS tools will now lookup required HMS configuration
(hive_metastore_uris and hive_metastore_sasl_enabled) from the master
when it is not provided via command line flags. Looking up the configs
on the master is more ergonomic and less error-prone.

Change-Id: Ibf580384a1612174a75142295d7f604cd6fabfe1
Reviewed-on: http://gerrit.cloudera.org:8080/11036
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d2afaec6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d2afaec6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d2afaec6

Branch: refs/heads/master
Commit: d2afaec640b062d8ecf7de432b3a6618265a1673
Parents: 85a1808
Author: Dan Burkert 
Authored: Mon Jul 23 16:49:02 2018 -0700
Committer: Dan Burkert 
Committed: Wed Aug 1 19:46:55 2018 +

--
 src/kudu/hms/hms_catalog.cc|  6 +-
 src/kudu/mini-cluster/external_mini_cluster.cc |  8 +-
 src/kudu/tools/kudu-tool-test.cc   | 81 +---
 src/kudu/tools/tool_action_common.cc   | 27 +--
 src/kudu/tools/tool_action_common.h| 16 
 src/kudu/tools/tool_action_hms.cc  | 85 +++--
 6 files changed, 158 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d2afaec6/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index deabb13..ad62513 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -54,9 +54,9 @@ using strings::Substitute;
 
 DEFINE_string(hive_metastore_uris, "",
   "Address of the Hive Metastore instance(s). The provided port 
must be for the HMS "
-  "Thrift service. If a port isn't provided, defaults to 9083. If 
the HMS is deployed "
-  "in an HA configuration, multiple comma-separated addresses can 
be supplied. If not "
-  "set, the Kudu master will not send Kudu table catalog updates 
to Hive. The "
+  "Thrift service. If a port is not provided, defaults to 9083. If 
the HMS is deployed "
+  "in an HA configuration, multiple comma-separated addresses 
should be supplied. "
+  "If not set, the Kudu master will not send Kudu table catalog 
updates to Hive. The "
   "configured value must match the Hive hive.metastore.uris 
configuration.");
 DEFINE_validator(hive_metastore_uris, ::hms::HmsCatalog::ValidateUris);
 TAG_FLAG(hive_metastore_uris, experimental);

http://git-wip-us.apache.org/repos/asf/kudu/blob/d2afaec6/src/kudu/mini-cluster/external_mini_cluster.cc
--
diff --git a/src/kudu/mini-cluster/external_mini_cluster.cc 
b/src/kudu/mini-cluster/external_mini_cluster.cc
index a188ad5..3d72375 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.cc
+++ b/src/kudu/mini-cluster/external_mini_cluster.cc
@@ -392,8 +392,9 @@ Status ExternalMiniCluster::StartMasters() {
 opts.rpc_bind_address = master_rpc_addrs[i];
 if (opts_.hms_mode == HmsMode::ENABLE_METASTORE_INTEGRATION) {
   opts.extra_flags.emplace_back(Substitute("--hive_metastore_uris=$0", 
hms_->uris()));
-  
opts.extra_flags.emplace_back(Substitute("--hive_metastore_sasl_enabled=$0",
-   opts_.enable_kerberos));
+  if (opts_.enable_kerberos) {
+opts.extra_flags.emplace_back("--hive_metastore_sasl_enabled=true");
+  }
 }
 opts.logtostderr = opts_.logtostderr;
 
@@ -884,8 +885,7 @@ void ExternalDaemon::SetExePath(string exe) {
 void ExternalDaemon::SetMetastoreIntegration(const string& hms_uris,
  bool enable_kerberos) {
   opts_.extra_flags.emplace_back(Substitute("--hive_metastore_uris=$0", 
hms_uris));
-  opts_.extra_flags.emplace_back(Substitute("--hive_metastore_sasl_enabled=$0",
-enable_kerberos));
+  
opts_.extra_flags.emplace_back(Substitute("--hive_metastore_sasl_enabled=$0", 
enable_kerberos));
 }
 
 Status ExternalDaemon::Pause() {

http://git-wip-us.apache.org/repos/asf/kudu/blob/d2afaec6/src/kudu/tools/kudu-tool-test.cc
--
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 7ee06ce..0a5743a 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -126,6 +126,7 @@
 #include "kudu/util/test_util.h"
 #include "kudu/util/url-coding.h"
 
+DECLARE_bool(hive_metastore_sasl_enabled);
 DECLARE_string(block_manager);
 DECLARE_string(hive_metastore_uris);

[4/4] kudu git commit: hms-tool: lookup master addresses config from master

[danburkert]

hms-tool: lookup master addresses config from master

The HMS check and fix tools currently use the master addresses
configuration provided on the command line to validate and fix metadata
in HMS table entries. This only works correctly if the administrator
inputs the master addresses exactly as its configured on the masters. If
the hostports are reordered or use a slightly different format which
resolves to the same addresses, then metadata will be flagged as stale
and rewritten unnecessarily. As an example, the administrator could pass
'localhost' as the master address if they ran it on the master node, but
this wouldn't be appropriate to write into the HMS.

This commit changes the handling so that the tools use the master
addresses returned from the ConnectToCluster mechanism already present
in the client, which should match the master addresses config of the
master exactly.

Change-Id: If170cebe6e5d7fa05fbd7faf18755aa57bdfeeec
Reviewed-on: http://gerrit.cloudera.org:8080/11083
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b2c64289
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b2c64289
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b2c64289

Branch: refs/heads/master
Commit: b2c6428909f468fa7dc106f601ee8c790060ac99
Parents: f847726
Author: Dan Burkert 
Authored: Mon Jul 30 10:56:12 2018 -0700
Committer: Dan Burkert 
Committed: Wed Aug 1 19:47:32 2018 +

--
 src/kudu/client/client-internal.cc   | 12 +++-
 src/kudu/client/client-internal.h|  8 +++-
 src/kudu/client/client.h | 24 +---
 src/kudu/tools/tool_action_common.cc |  4 
 src/kudu/tools/tool_action_common.h  |  3 +++
 src/kudu/tools/tool_action_hms.cc| 26 +-
 6 files changed, 55 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/b2c64289/src/kudu/client/client-internal.cc
--
diff --git a/src/kudu/client/client-internal.cc 
b/src/kudu/client/client-internal.cc
index 10def55..335a23a 100644
--- a/src/kudu/client/client-internal.cc
+++ b/src/kudu/client/client-internal.cc
@@ -35,6 +35,7 @@
 #include "kudu/client/master_rpc.h"
 #include "kudu/client/meta_cache.h"
 #include "kudu/client/schema.h"
+#include "kudu/common/common.pb.h"
 #include "kudu/common/partition.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/wire_protocol.h"
@@ -46,6 +47,7 @@
 #include "kudu/master/master.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/master/master.proxy.h"
+#include "kudu/rpc/connection.h"
 #include "kudu/rpc/messenger.h"
 #include "kudu/rpc/request_tracker.h"
 #include "kudu/rpc/rpc_controller.h"
@@ -60,7 +62,6 @@
 #include "kudu/util/net/sockaddr.h"
 #include "kudu/util/pb_util.h"
 #include "kudu/util/thread_restrictions.h"
-#include "kudu/rpc/connection.h"
 
 using std::pair;
 using std::set;
@@ -702,6 +703,10 @@ void KuduClient::Data::ConnectedToClusterCb(
 
 if (status.ok()) {
   leader_master_hostport_ = HostPort(leader_hostname, leader_addr.port());
+  master_hostports_.clear();
+  for (const auto& hostport : connect_response.master_addrs()) {
+master_hostports_.emplace_back(HostPort(hostport.host(), 
hostport.port()));
+  }
   master_proxy_.reset(new MasterServiceProxy(messenger_, leader_addr, 
leader_hostname));
   master_proxy_->set_user_credentials(user_credentials_);
 }
@@ -822,6 +827,11 @@ HostPort KuduClient::Data::leader_master_hostport() const {
   return leader_master_hostport_;
 }
 
+const vector& KuduClient::Data::master_hostports() const {
+  std::lock_guard l(leader_master_lock_);
+  return master_hostports_;
+}
+
 shared_ptr KuduClient::Data::master_proxy() const {
   std::lock_guard l(leader_master_lock_);
   return master_proxy_;

http://git-wip-us.apache.org/repos/asf/kudu/blob/b2c64289/src/kudu/client/client-internal.h
--
diff --git a/src/kudu/client/client-internal.h 
b/src/kudu/client/client-internal.h
index e77e3bf..467d5ec 100644
--- a/src/kudu/client/client-internal.h
+++ b/src/kudu/client/client-internal.h
@@ -191,6 +191,8 @@ class KuduClient::Data {
 
   HostPort leader_master_hostport() const;
 
+  const std::vector& master_hostports() const;
+
   uint64_t GetLatestObservedTimestamp() const;
 
   void UpdateLatestObservedTimestamp(uint64_t timestamp);
@@ -254,6 +256,10 @@ class KuduClient::Data {
   // ConnectToClusterAsync.
   HostPort leader_master_hostport_;
 
+  // The master RPC host ports as configured on the most recently connected to
+  // leader master in ConnectedToClusterCb.
+  std::vector master_hostports_;
+
   // Proxy to the leader master.

[3/4] kudu git commit: hms precheck tool

[danburkert]

hms precheck tool

This commit adds a 'kudu hms precheck' tool that is meant to be used
before enabling the HMS integration on an existing cluster. The tool
checks for tables whose names are not unique when compared with Hive's
case-insensitive identifier rules. These conflicting tables would cause
the master to fail to startup after enabling the Hive Metastore
integration.

Change-Id: Ia7b23cb802b6434eba7e38443d3361ef70e6543e
Reviewed-on: http://gerrit.cloudera.org:8080/11040
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f8477267
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f8477267
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f8477267

Branch: refs/heads/master
Commit: f847726780498b1a0ef1b9a7d95a50b4f327b36d
Parents: d2afaec
Author: Dan Burkert 
Authored: Tue Jul 24 14:36:13 2018 -0700
Committer: Dan Burkert 
Committed: Wed Aug 1 19:47:03 2018 +

--
 src/kudu/tools/kudu-tool-test.cc  | 79 +++--
 src/kudu/tools/tool_action_hms.cc | 80 +-
 2 files changed, 144 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f8477267/src/kudu/tools/kudu-tool-test.cc
--
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 0a5743a..8293ef2 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -2486,7 +2486,7 @@ TEST_P(ToolTestKerberosParameterized, 
TestCheckAndAutomaticFixHmsMetadata) {
   vector kudu_tables;
   kudu_client->ListTables(_tables);
   std::sort(kudu_tables.begin(), kudu_tables.end());
-  ASSERT_EQ(kudu_tables, vector({
+  ASSERT_EQ(vector({
 "default.control",
 "default.inconsistent_master_addrs",
 "default.inconsistent_name_hms",
@@ -2497,7 +2497,7 @@ TEST_P(ToolTestKerberosParameterized, 
TestCheckAndAutomaticFixHmsMetadata) {
 "default.legacy_managed",
 "default.uppercase",
 "my_db.table",
-  }));
+  }), kudu_tables);
 }
 
 // Test HMS inconsistencies that must be manually fixed.
@@ -2624,14 +2624,85 @@ TEST_P(ToolTestKerberosParameterized, 
TestCheckAndManualFixHmsMetadata) {
   vector kudu_tables;
   kudu_client->ListTables(_tables);
   std::sort(kudu_tables.begin(), kudu_tables.end());
-  ASSERT_EQ(kudu_tables, vector({
+  ASSERT_EQ(vector({
 "default.conflicting_legacy_table",
 "default.duplicate_hms_tables",
 "default.hive_compatible_name",
 "default.non_conflicting_legacy_table",
 "default.with_database",
 "non_existent_database.table",
-  }));
+  }), kudu_tables);
+}
+
+TEST_F(ToolTest, TestHmsPrecheck) {
+  ExternalMiniClusterOptions opts;
+  opts.hms_mode = HmsMode::ENABLE_HIVE_METASTORE;
+  NO_FATALS(StartExternalMiniCluster(std::move(opts)));
+
+  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+
+  shared_ptr client;
+  ASSERT_OK(cluster_->CreateClient(nullptr, ));
+
+  // Create test tables.
+  for (const string& table_name : {
+  "a.b",
+  "foo.bar",
+  "FOO.bar",
+  "foo.BAR",
+  "fuzz",
+  "FUZZ",
+  "a.b!",
+  "A.B!",
+  }) {
+  ASSERT_OK(CreateKuduTable(client, table_name));
+  }
+
+  // Run the precheck tool. It should complain about the conflicting tables.
+  string out;
+  string err;
+  Status s = RunActionStdoutStderrString(Substitute("hms precheck $0", 
master_addr), , );
+  ASSERT_FALSE(s.ok());
+  ASSERT_STR_CONTAINS(err, "found tables in Kudu with case-conflicting names");
+  ASSERT_STR_CONTAINS(out, "foo.bar");
+  ASSERT_STR_CONTAINS(out, "FOO.bar");
+  ASSERT_STR_CONTAINS(out, "foo.BAR");
+
+  // It should not complain about tables which don't have conflicting names.
+  ASSERT_STR_NOT_CONTAINS(out, "a.b");
+
+  // It should not complain about tables which have Hive-incompatible names.
+  ASSERT_STR_NOT_CONTAINS(out, "fuzz");
+  ASSERT_STR_NOT_CONTAINS(out, "FUZZ");
+  ASSERT_STR_NOT_CONTAINS(out, "a.b!");
+  ASSERT_STR_NOT_CONTAINS(out, "A.B!");
+
+  // Rename the conflicting tables. Use the rename table tool to match the 
actual workflow.
+  NO_FATALS(RunActionStdoutNone(Substitute("table rename_table $0 FOO.bar 
foo.bar2", master_addr)));
+  NO_FATALS(RunActionStdoutNone(Substitute("table rename_table $0 foo.BAR 
foo.bar3", master_addr)));
+
+  // Precheck should now pass, and the cluster should upgrade succesfully.
+  NO_FATALS(RunActionStdoutNone(Substitute("hms precheck $0", master_addr)));
+
+  // Enable the HMS integration.
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
+  cluster_->EnableMetastoreIntegration();
+  ASSERT_OK(cluster_->Restart());
+
+  // Sanity-check the tables.
+  vector tables;
+  

kudu git commit: KUDU-2509 fix use-after-free in case of WAL replay error

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.7.x cf21f3673 -> 7fba099d5


KUDU-2509 fix use-after-free in case of WAL replay error

Fixed use-after-free mistake in case of a failure to apply a pending
commit message from the WAL while bootstrapping a tablet.

Also, a repro scenario to expose the use-after-free condition is added.
Prior to the fix, the repro scenario would crash with SIGSEGV on Linux
or with SIGBUS on OS X (at least for DEBUG builds).

Change-Id: I11373b1cc34d9e2e0181bee2d3841b49022218ed
Reviewed-on: http://gerrit.cloudera.org:8080/10997
Tested-by: Alexey Serbin 
Reviewed-by: Adar Dembo 
(cherry picked from commit 6b429e8a42ad9fb12a97cc26e33ca19ac2626533)
Reviewed-on: http://gerrit.cloudera.org:8080/11063
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7fba099d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7fba099d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7fba099d

Branch: refs/heads/branch-1.7.x
Commit: 7fba099d553ed7b3e56c1d8e89dcc0bae4397616
Parents: cf21f36
Author: Alexey Serbin 
Authored: Thu Jul 19 21:05:54 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Jul 31 23:44:10 2018 +

--
 src/kudu/tablet/tablet_bootstrap-test.cc | 75 ++-
 src/kudu/tablet/tablet_bootstrap.cc  |  2 +-
 2 files changed, 74 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7fba099d/src/kudu/tablet/tablet_bootstrap-test.cc
--
diff --git a/src/kudu/tablet/tablet_bootstrap-test.cc 
b/src/kudu/tablet/tablet_bootstrap-test.cc
index e8131e0..ed32517 100644
--- a/src/kudu/tablet/tablet_bootstrap-test.cc
+++ b/src/kudu/tablet/tablet_bootstrap-test.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "kudu/consensus/log-test-base.h"
+#include "kudu/tablet/tablet_bootstrap.h"
 
 #include 
 #include 
@@ -35,7 +35,9 @@
 #include "kudu/clock/logical_clock.h"
 #include "kudu/common/common.pb.h"
 #include "kudu/common/iterator.h"
+#include "kudu/common/partial_row.h"
 #include "kudu/common/partition.h"
+#include "kudu/common/row_operations.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/timestamp.h"
 #include "kudu/common/wire_protocol-test-util.h"
@@ -45,6 +47,7 @@
 #include "kudu/consensus/consensus.pb.h"
 #include "kudu/consensus/consensus_meta.h"
 #include "kudu/consensus/consensus_meta_manager.h"
+#include "kudu/consensus/log-test-base.h"
 #include "kudu/consensus/log.h"
 #include "kudu/consensus/log_anchor_registry.h"
 #include "kudu/consensus/log_reader.h"
@@ -68,7 +71,6 @@
 #include "kudu/tablet/tablet-test-util.h"
 #include "kudu/tablet/tablet.h"
 #include "kudu/tablet/tablet.pb.h"
-#include "kudu/tablet/tablet_bootstrap.h"
 #include "kudu/tablet/tablet_metadata.h"
 #include "kudu/tablet/tablet_replica.h"
 #include "kudu/tserver/tserver.pb.h"
@@ -676,5 +678,74 @@ TEST_F(BootstrapTest, 
TestConsensusOnlyOperationOutOfOrderTimestamp) {
   ASSERT_EQ(1, results.size());
 }
 
+// Regression test for KUDU-2509. There was a use-after-free bug that sometimes
+// lead to SIGSEGV while replaying the WAL. This scenario would crash or
+// at least UB sanitizer would report a warning if such condition exists.
+TEST_F(BootstrapTest, TestKudu2509) {
+  ASSERT_OK(BuildLog());
+
+  consensus::ReplicateRefPtr replicate = 
consensus::make_scoped_refptr_replicate(
+  new consensus::ReplicateMsg());
+  replicate->get()->set_op_type(consensus::WRITE_OP);
+  tserver::WriteRequestPB* batch_request = 
replicate->get()->mutable_write_request();
+  ASSERT_OK(SchemaToPB(schema_, batch_request->mutable_schema()));
+  batch_request->set_tablet_id(log::kTestTablet);
+
+  // This appends Insert(1) with op 10.10
+  const OpId insert_opid = MakeOpId(10, 10);
+  replicate->get()->mutable_id()->CopyFrom(insert_opid);
+  replicate->get()->set_timestamp(clock_->Now().ToUint64());
+  AddTestRowToPB(RowOperationsPB::INSERT, schema_, 10, 1,
+ "this is a test insert", 
batch_request->mutable_row_operations());
+  NO_FATALS(AppendReplicateBatch(replicate));
+
+  // This appends Mutate(1) with op 10.11. The operation would try to update
+  // a row having an extra column. This should fail since there hasn't been
+  // corresponding DDL operation committed yet.
+  const OpId mutate_opid = MakeOpId(10, 11);
+  batch_request->mutable_row_operations()->Clear();
+  replicate->get()->mutable_id()->CopyFrom(mutate_opid);
+  replicate->get()->set_timestamp(clock_->Now().ToUint64());
+  {
+// Modify the existing schema to add an extra row.
+SchemaBuilder builder(schema_);
+ASSERT_OK(builder.AddNullableColumn("string_val_extra", STRING));
+

[4/4] kudu git commit: hms-tool: refactor check tool and combine upgrade and fix

[danburkert]

hms-tool: refactor check tool and combine upgrade and fix

This commit combines the 'hms upgrade' and 'hms fix' tools. The tools
previously had overlapping responsibilities for checking some types of
inconsistencies. The 'fix' tool now has flags which can control whether
it attempts certain types of fixes:

  --drop_orphan_hms_tables=false
  --create_missing_hms_tables=true
  --fix_inconsistent_tables=true
  --upgrade_hms_tables=true

`drop_orphan_hms_tables` defaults to false since it deletes Hive tables,
and the tool can not rule-out that they are being used. Additionally, a
--dryrun flag is provided in order to print steps that would be taken
without actually making modifications.

The checking logic has been expanded to account for more potential
inconsistencies, and where possible the fix tool now can automatically
repair these inconsistencies.

Finally, the input prompt and default database functionality has been
removed in order to simplify the tool. Instead, the check tool will
issue hints including instructions for how to rename tables without a
database or with a Hive-incompatible name using the
'kudu table rename_table' tool. We can always add this functionality
back in the future if we determine it helps out users.

Change-Id: Ieee478002eb56a278cfd74255670baaf28109b8f
Reviewed-on: http://gerrit.cloudera.org:8080/11018
Tested-by: Dan Burkert 
Reviewed-by: Hao Hao 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8e40dfdb
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8e40dfdb
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8e40dfdb

Branch: refs/heads/master
Commit: 8e40dfdb97ee54906dfa519e28fbcffc290095ac
Parents: 660ee80
Author: Dan Burkert 
Authored: Fri Jul 6 17:04:10 2018 -0700
Committer: Dan Burkert 
Committed: Mon Jul 30 18:46:28 2018 +

--
 src/kudu/gutil/strings/escaping.h |  10 +-
 src/kudu/hms/hms_catalog.cc   |  14 +-
 src/kudu/hms/hms_catalog.h|  17 +-
 src/kudu/tools/CMakeLists.txt |   7 +-
 src/kudu/tools/kudu-tool-test.cc  | 777 ++---
 src/kudu/tools/tool_action_hms.cc | 888 ++---
 6 files changed, 891 insertions(+), 822 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8e40dfdb/src/kudu/gutil/strings/escaping.h
--
diff --git a/src/kudu/gutil/strings/escaping.h 
b/src/kudu/gutil/strings/escaping.h
index c39bf8a..1f40f68 100644
--- a/src/kudu/gutil/strings/escaping.h
+++ b/src/kudu/gutil/strings/escaping.h
@@ -142,7 +142,15 @@ bool CUnescape(const StringPiece& source, std::string* 
dest, std::string* error)
 
 // A version with no error reporting.
 inline bool CUnescape(const StringPiece& source, std::string* dest) {
-  return CUnescape(source, dest, NULL);
+  return CUnescape(source, dest, nullptr);
+}
+
+// A version which CHECK fails if the string can not be unescaped.
+inline std::string CUnescapeOrDie(const StringPiece& source) {
+  std::string dest;
+  std::string err;
+  CHECK(CUnescape(source, , )) << err;
+  return dest;
 }
 
 // --

http://git-wip-us.apache.org/repos/asf/kudu/blob/8e40dfdb/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index 11faec0..deabb13 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -151,13 +151,19 @@ Status HmsCatalog::CreateTable(const string& id,
 }
 
 Status HmsCatalog::DropTable(const string& id, const string& name) {
-  Slice hms_database;
-  Slice hms_table;
-  RETURN_NOT_OK(ParseTableName(name, _database, _table));
-
   hive::EnvironmentContext env_ctx = EnvironmentContext();
   env_ctx.properties.insert(make_pair(HmsClient::kKuduTableIdKey, id));
+  return DropTable(name, env_ctx);
+}
 
+Status HmsCatalog::DropLegacyTable(const string& name) {
+  return DropTable(name, EnvironmentContext());
+}
+
+Status HmsCatalog::DropTable(const string& name, const 
hive::EnvironmentContext& env_ctx) {
+  Slice hms_database;
+  Slice hms_table;
+  RETURN_NOT_OK(ParseTableName(name, _database, _table));
   return Execute([&] (HmsClient* client) {
 return client->DropTable(hms_database.ToString(), hms_table.ToString(), 
env_ctx);
   });

http://git-wip-us.apache.org/repos/asf/kudu/blob/8e40dfdb/src/kudu/hms/hms_catalog.h
--
diff --git a/src/kudu/hms/hms_catalog.h b/src/kudu/hms/hms_catalog.h
index f75ee3e..ebb1f41 100644
--- a/src/kudu/hms/hms_catalog.h
+++ b/src/kudu/hms/hms_catalog.h
@@ -75,8 +75,17 @@ class HmsCatalog {
   // This method will fail if the HMS is unreachable, if the table does 

[2/4] kudu git commit: [kudu-admin-test] improvements on error reporting

[danburkert]

[kudu-admin-test] improvements on error reporting

This changelist introduces improvements on logging in case
if an assertion triggers in the rebalancer-related scenarios.

I found it's hard to troubleshoot the flakiness reported lately
in the RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove
scenario.  With hindsight, it turned out that improved logging
on assertions would help a lot.

Change-Id: Ibfa74d5a78e89bf7ff1a0e914384999466460145
Reviewed-on: http://gerrit.cloudera.org:8080/11046
Reviewed-by: Andrew Wong 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/660ee80b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/660ee80b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/660ee80b

Branch: refs/heads/master
Commit: 660ee80bb4c0f81cb6f0a08365a7e2abd0a3e409
Parents: 7e733a7
Author: Alexey Serbin 
Authored: Tue Jul 24 20:20:00 2018 -0700
Committer: Alexey Serbin 
Committed: Mon Jul 30 15:06:44 2018 +

--
 src/kudu/tools/kudu-admin-test.cc | 80 --
 1 file changed, 47 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/660ee80b/src/kudu/tools/kudu-admin-test.cc
--
diff --git a/src/kudu/tools/kudu-admin-test.cc 
b/src/kudu/tools/kudu-admin-test.cc
index 21bae3b..0408e35 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -48,7 +48,6 @@
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
-#include "kudu/gutil/strings/util.h"
 #include "kudu/integration-tests/cluster_itest_util.h"
 #include "kudu/integration-tests/cluster_verifier.h"
 #include "kudu/integration-tests/test_workload.h"
@@ -105,6 +104,7 @@ using std::atomic;
 using std::back_inserter;
 using std::copy;
 using std::deque;
+using std::endl;
 using std::ostringstream;
 using std::string;
 using std::thread;
@@ -1331,6 +1331,14 @@ TEST_F(AdminCliTest, TestListTablesDetail) {
   }
 }
 
+static string ToolRunInfo(const Status& s, const string& out, const string& 
err) {
+  ostringstream str;
+  str << s.ToString() << endl;
+  str << "stdout: " << out << endl;
+  str << "stderr: " << err << endl;
+  return str.str();
+}
+
 // Make sure the rebalancer doesn't start if a tablet server is down.
 class RebalanceStartCriteriaTest :
 public AdminCliTest,
@@ -1359,13 +1367,14 @@ TEST_P(RebalanceStartCriteriaTest, TabletServerIsDown) {
 ts->Shutdown();
   }
 
+  string out;
   string err;
   Status s = RunKuduTool({
 "cluster",
 "rebalance",
 cluster_->master()->bound_rpc_addr().ToString()
-  }, nullptr, );
-  ASSERT_TRUE(s.IsRuntimeError()) << s.ToString();
+  }, , );
+  ASSERT_TRUE(s.IsRuntimeError()) << ToolRunInfo(s, out, err);
   const auto err_msg_pattern = Substitute(
   "Illegal state: tablet server .* \\($0\\): "
   "unacceptable health status UNAVAILABLE",
@@ -1501,7 +1510,7 @@ TEST_P(RebalanceParamTest, Rebalance) {
 string out;
 string err;
 const Status s = RunKuduTool(tool_args, , );
-ASSERT_TRUE(s.ok()) << s.ToString() << ":" << err;
+ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
 ASSERT_STR_CONTAINS(out, "rebalancing is complete: cluster is balanced")
 << "stderr: " << err;
   }
@@ -1512,7 +1521,7 @@ TEST_P(RebalanceParamTest, Rebalance) {
 string out;
 string err;
 const Status s = RunKuduTool(tool_args, , );
-ASSERT_TRUE(s.ok()) << s.ToString() << ":" << err;
+ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
 ASSERT_STR_CONTAINS(out,
 "rebalancing is complete: cluster is balanced (moved 0 replicas)")
 << "stderr: " << err;
@@ -1532,7 +1541,7 @@ TEST_P(RebalanceParamTest, Rebalance) {
 string out;
 string err;
 const Status s = RunKuduTool(tool_args, , );
-ASSERT_TRUE(s.ok()) << s.ToString() << ":" << err;
+ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
 ASSERT_STR_CONTAINS(out, "rebalancing is complete: cluster is balanced")
 << "stderr: " << err;
 // The cluster was un-balanced, so many replicas should have been moved.
@@ -1570,6 +1579,8 @@ class RebalancingTest :
   }
 
  protected:
+  static const char* const kExitOnSignalStr;
+
   void Prepare(const vector& extra_tserver_flags = {},
const vector& extra_master_flags = {}) {
 copy(extra_tserver_flags.begin(), extra_tserver_flags.end(),
@@ -1616,6 +1627,7 @@ class RebalancingTest :
   vector tserver_flags_;
   vector master_flags_;
 };
+const char* const RebalancingTest::kExitOnSignalStr = "kudu: process exited on 
signal";
 
 // Make sure the rebalancer is able to do its job if running concurrently
 // with DDL activity on the cluster.
@@ -1781,7 

[1/4] kudu git commit: [util] update on ParseVersion()

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 34b058b8b -> 8e40dfdb9


[util] update on ParseVersion()

This changelist updates implementation of the kudu::ParseVersion()
utility function to recognize and successfully parse version strings
like '1.8.0-cdh6.x-SNAPSHOT'.

Version::ToString() implementation changed to return
'canonical' representation of the parsed version string.  Added
corresponding unit tests.

This patch also fixes KUDU-2511.

This is a follow-up for c12e63528a2f2c265136e702a48cbc6c93b7f2fd.

Change-Id: I98b200d0529cb4471e4e855b88bfdb2ff2f346bf
Reviewed-on: http://gerrit.cloudera.org:8080/11060
Tested-by: Alexey Serbin 
Reviewed-by: Andrew Wong 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7e733a73
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7e733a73
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7e733a73

Branch: refs/heads/master
Commit: 7e733a73652db68342859d03d02763ef5e3de79d
Parents: 34b058b
Author: Alexey Serbin 
Authored: Wed Jul 25 22:07:11 2018 -0700
Committer: Alexey Serbin 
Committed: Mon Jul 30 01:55:52 2018 +

--
 src/kudu/util/version_util-test.cc | 88 +++--
 src/kudu/util/version_util.cc  | 29 ++-
 src/kudu/util/version_util.h   | 11 +++--
 3 files changed, 98 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7e733a73/src/kudu/util/version_util-test.cc
--
diff --git a/src/kudu/util/version_util-test.cc 
b/src/kudu/util/version_util-test.cc
index 54e8e76..3cb39b9 100644
--- a/src/kudu/util/version_util-test.cc
+++ b/src/kudu/util/version_util-test.cc
@@ -14,53 +14,109 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
 #include "kudu/util/version_util.h"
 
 #include 
+#include 
 #include 
 
 #include 
 
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
-#include "kudu/util/test_util.h"
+#include "kudu/util/version_info.h"
 
 using std::string;
+using std::pair;
 using std::vector;
 
 namespace kudu {
 
-class VersionUtilTest : public KuduTest {};
-
-TEST_F(VersionUtilTest, TestVersion) {
-  const vector good_test_cases = {
-{ "0.0.0", 0, 0, 0, "" },
-{ "1.0.0", 1, 0, 0, "" },
-{ "1.1.0", 1, 1, 0, "" },
-{ "1.1.1", 1, 1, 1, "" },
-{ "1.10.100-1000", 1, 10, 100, "1000" },
-{ "1.2.3-SNAPSHOT", 1, 2, 3, "SNAPSHOT" },
+TEST(VersionUtilTest, TestVersion) {
+  const vector> good_test_cases = {
+{ { "0.0.0", 0, 0, 0, "" }, "0.0.0" },
+{ { "1.0.0", 1, 0, 0, "" }, "1.0.0" },
+{ { "1.1.0", 1, 1, 0, "" }, "1.1.0" },
+{ { "1.1.1", 1, 1, 1, "" }, "1.1.1" },
+{ { "1.1.1-", 1, 1, 1, "" }, "1.1.1" },
+{ { "1.1.1-0-1-2", 1, 1, 1, "0-1-2" }, "1.1.1-0-1-2" },
+{ { "1.10.100-1000.0", 1, 10, 100, "1000.0" }, "1.10.100-1000.0" },
+{ { "1.2.3-SNAPSHOT", 1, 2, 3, "SNAPSHOT" }, "1.2.3-SNAPSHOT" },
+{ { "1.8.0-x-SNAPSHOT", 1, 8, 0, "x-SNAPSHOT" }, "1.8.0-x-SNAPSHOT" },
+{ { "0.1.2-a-b-c-d", 0, 1, 2, "a-b-c-d" }, "0.1.2-a-b-c-d" },
+{ { " 0 .1 .2 -x", 0, 1, 2, "x" }, "0.1.2-x" },
+{ { " 0 . 1 .  2 -x ", 0, 1, 2, "x" }, "0.1.2-x" },
+{ { "+1.+2.+3-6", 1, 2, 3, "6" }, "1.2.3-6" },
+{ { " +1 .  +2 .   +3 -100 .. ", 1, 2, 3, "100 .." }, "1.2.3-100 .." },
+// no octals: leading zeros are just chopped off
+{ { "00.01.010", 0, 1, 10, "" }, "0.1.10" },
+{ { "  0.1.2suffix  ", 0, 1, 2, "---suffix" }, "0.1.2suffix" },
+{ { "0.1.2- - -x- -y- ", 0, 1, 2, " - -x- -y-" }, "0.1.2- - -x- -y-" },
   };
 
-  Version v;
   for (const auto& test_case : good_test_cases) {
-ASSERT_OK(ParseVersion(test_case.raw_version, ));
-EXPECT_EQ(test_case, v);
+const auto& version = test_case.first;
+const auto& canonical_str = test_case.second;
+Version v;
+ASSERT_OK(ParseVersion(version.raw_version, ));
+EXPECT_EQ(version, v);
+EXPECT_EQ(canonical_str, v.ToString());
   }
 
   const vector bad_test_cases = {
 "",
+" ",
+"-",
+" -",
+"--",
+" - - ",
+"..",
+"0.1",
+"0.1.",
+"0.1.+",
+"+ 0.+ 1.+ 2",
+"0.1.+-woops",
+"0.1.2+",
+"1..1",
+".0.1",
+" . . ",
+"-0.1.2-013",
+"0.-1.2-013",
+"0.1.-2-013",
+"1000,000.2999,999.999",
+"1e10.0.1",
+"1e+10.0.1",
+"1e-1.0.1",
+"1E+1.2.3",
+"1E-5.0.1",
 "foo",
 "foo.1.0",
+"a.b.c",
+"0.x1.x2",
+"0x0.0x1.0x2",
 "1.bar.0",
 "1.0.foo",
 "1.0foo.bar",
 "foo5-1.4.3",
   };
 
-  for (const auto& test_case : bad_test_cases) {
-ASSERT_TRUE(ParseVersion(test_case, ).IsInvalidArgument());
+  for (const auto& input_str : bad_test_cases) {
+

[3/4] kudu git commit: hms-tool: refactor check tool and combine upgrade and fix

[danburkert]

http://git-wip-us.apache.org/repos/asf/kudu/blob/8e40dfdb/src/kudu/tools/tool_action_hms.cc
--
diff --git a/src/kudu/tools/tool_action_hms.cc 
b/src/kudu/tools/tool_action_hms.cc
index 6273ed2..db90ee3 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -19,6 +19,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -32,21 +33,35 @@
 #include "kudu/client/shared_ptr.h"
 #include "kudu/common/schema.h"
 #include "kudu/gutil/map-util.h"
-#include "kudu/gutil/strings/join.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
-#include "kudu/gutil/strings/util.h"
 #include "kudu/hms/hive_metastore_types.h"
 #include "kudu/hms/hms_catalog.h"
 #include "kudu/hms/hms_client.h"
 #include "kudu/tools/tool_action.h"
 #include "kudu/tools/tool_action_common.h"
 #include "kudu/util/monotime.h"
+#include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 
-DECLARE_int64(timeout_ms); // defined in tool_action_common
+DECLARE_bool(force);
+DECLARE_int64(timeout_ms);
 DECLARE_string(hive_metastore_uris);
 
+DEFINE_bool(dryrun, false,
+"Print a message for each fix, but do not make modifications to Kudu or 
the Hive Metastore.");
+DEFINE_bool(drop_orphan_hms_tables, false,
+"Drop orphan Hive Metastore tables which refer to non-existent Kudu 
tables.");
+DEFINE_bool(create_missing_hms_tables, true,
+"Create a Hive Metastore table for each Kudu table which is missing one.");
+DEFINE_bool(fix_inconsistent_tables, true,
+"Fix tables whose Kudu and Hive Metastore metadata differ. If the table 
name is "
+"different, the table is renamed in Kudu to match the HMS. If the columns "
+"or other metadata is different the HMS is updated to match Kudu.");
+DEFINE_bool(upgrade_hms_tables, true,
+"Upgrade Hive Metastore tables from the legacy Impala metadata format to 
the "
+"new Kudu metadata format.");
+
 namespace kudu {
 namespace tools {
 
@@ -55,11 +70,11 @@ using client::KuduClientBuilder;
 using client::KuduTable;
 using client::KuduTableAlterer;
 using client::sp::shared_ptr;
-using hms::HmsClient;
 using hms::HmsCatalog;
-using std::cin;
+using hms::HmsClient;
 using std::cout;
 using std::endl;
+using std::make_pair;
 using std::ostream;
 using std::pair;
 using std::string;
@@ -69,143 +84,16 @@ using std::vector;
 using strings::Split;
 using strings::Substitute;
 
-DEFINE_bool(enable_input, true,
-"Whether to enable user input for renaming tables that have Hive"
-"incompatible names.");
-
-// The key is the table ID. The value is a pair of a table in Kudu and a
-// vector of tables in the HMS that all share the same table ID.
-typedef unordered_map, 
vector>> TablesMap;
-
-const char* const kDefaultDatabaseArg = "default_database";
-const char* const kDefaultDatabaseArgDesc = "The database that non-Impala Kudu 
"
-"table should reside in";
-const char* const kInvalidNameError = "is not a valid object name";
-
-// Returns a map that contains all Kudu tables in the HMS. Its key is
-// the Kudu table name and its value is the corresponding HMS table.
-unordered_map RetrieveTablesMap(vector 
hms_tables) {
-  unordered_map hms_tables_map;
-  for (auto& hms_table : hms_tables) {
-const string& storage_handler = 
hms_table.parameters[HmsClient::kStorageHandlerKey];
-if (storage_handler == HmsClient::kLegacyKuduStorageHandler) {
-  
hms_tables_map.emplace(hms_table.parameters[HmsClient::kLegacyKuduTableNameKey],
 hms_table);
-} else if (storage_handler == HmsClient::kKuduStorageHandler) {
-  hms_tables_map.emplace(Substitute("$0.$1", hms_table.dbName, 
hms_table.tableName), hms_table);
-}
-  }
-  return hms_tables_map;
-}
-
-string RenameHiveIncompatibleTable(const string& table_name) {
-  cout << Substitute("Table $0 is not hive compatible.", table_name) << endl;
-  cout << "Please input a new table name: ";
-  string new_table_name;
-  getline(cin, new_table_name);
-  return new_table_name;
-}
-
 // Only alter the table in Kudu but not in the Hive Metastore.
-Status AlterKuduTableOnly(KuduClient* kudu_client,
-  const string& name,
-  const string& new_name) {
+Status RenameTableInKuduCatalog(KuduClient* kudu_client,
+const string& name,
+const string& new_name) {
   unique_ptr alterer(kudu_client->NewTableAlterer(name));
   SetAlterExternalCatalogs(alterer.get(), false);
   return alterer->RenameTo(new_name)
 ->Alter();
 }
 
-// Alter legacy tables (which includes non-Impala tables, Impala 
managed/external
-// tables) to follow the format 'database_name.table_name' in table naming in 
Kudu.
-// Also, create HMS entries for non-Impala tables.
-//
-// Note that non-Impala tables name should conform to Hive naming 

[1/2] kudu git commit: Add kudu-backup module to Maven build

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 0418b225e -> 34b058b8b


Add kudu-backup module to Maven build

Change Base64 encode/decode to java 7 compatible versions to fix sniffer
errors.

Change-Id: Ia1b34b733d8d85106060068e611be7eecc0717f2
Reviewed-on: http://gerrit.cloudera.org:8080/11059
Reviewed-by: Grant Henke 
Tested-by: Grant Henke 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fb79471c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fb79471c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fb79471c

Branch: refs/heads/master
Commit: fb79471c26ddd5d325febd2d40456ca8e83b6f04
Parents: 0418b22
Author: Tony Foerster 
Authored: Thu Jul 26 11:34:40 2018 -0500
Committer: Grant Henke 
Committed: Thu Jul 26 21:50:40 2018 +

--
 java/kudu-backup/pom.xml| 9 -
 .../main/scala/org/apache/kudu/backup/TableMetadata.scala   | 8 
 java/pom.xml| 1 +
 3 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fb79471c/java/kudu-backup/pom.xml
--
diff --git a/java/kudu-backup/pom.xml b/java/kudu-backup/pom.xml
index 7f0e1a5..37aefc4 100644
--- a/java/kudu-backup/pom.xml
+++ b/java/kudu-backup/pom.xml
@@ -116,6 +116,13 @@
 test
 
 
+org.apache.kudu
+kudu-client
+${project.version}
+test-jar
+test
+
+
 junit
 junit
 ${junit.version}
@@ -162,7 +169,7 @@
  
https://www.xolstice.org/protobuf-maven-plugin/usage.html -->
 true
 src/main/protobuf
-
com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}
+
com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
 
 
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/fb79471c/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
--
diff --git 
a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala 
b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
index 9110e5f..9574943 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/TableMetadata.scala
@@ -17,13 +17,13 @@
 package org.apache.kudu.backup
 
 import java.math.BigDecimal
-import java.util.Base64
 
 import com.google.protobuf.StringValue
+import org.apache.commons.net.util.Base64
 import org.apache.kudu.backup.Backup._
 import org.apache.kudu.ColumnSchema.{ColumnSchemaBuilder, 
CompressionAlgorithm, Encoding}
 import org.apache.kudu.ColumnTypeAttributes.ColumnTypeAttributesBuilder
-import org.apache.kudu.client.{CreateTableOptions, KuduTable, PartialRow}
+import org.apache.kudu.client.{Bytes, CreateTableOptions, KuduTable, 
PartialRow}
 import org.apache.kudu.{ColumnSchema, Schema, Type}
 import org.apache.yetus.audience.{InterfaceAudience, InterfaceStability}
 
@@ -223,7 +223,7 @@ object TableMetadata {
   case Type.STRING =>
 value.asInstanceOf[String]
   case Type.BINARY =>
-Base64.getEncoder.encodeToString(value.asInstanceOf[Array[Byte]])
+Base64.encodeBase64String(value.asInstanceOf[Array[Byte]])
   case Type.DECIMAL =>
 value.asInstanceOf[BigDecimal].toString // TODO: Explicitly control 
print format
   case _ => throw new IllegalArgumentException(s"Unsupported column type: 
$colType")
@@ -240,7 +240,7 @@ object TableMetadata {
   case Type.FLOAT => value.toFloat
   case Type.DOUBLE => value.toDouble
   case Type.STRING => value
-  case Type.BINARY => Base64.getDecoder.decode(value)
+  case Type.BINARY => Base64.decodeBase64(value.asInstanceOf[Array[Byte]])
   case Type.DECIMAL => new BigDecimal(value) // TODO: Explicitly pass scale
   case _ => throw new IllegalArgumentException(s"Unsupported column type: 
$colType")
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/fb79471c/java/pom.xml
--
diff --git a/java/pom.xml b/java/pom.xml
index 4dbe193..29958d6 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -105,6 +105,7 @@
 
 
 
+kudu-backup
 kudu-client-tools
 kudu-client
 kudu-flume-sink

[2/2] kudu git commit: [DOCS] Fixed a broken link in administration.adoc

[danburkert]

[DOCS] Fixed a broken link in administration.adoc

Change-Id: I7ca94cc24b3038659b89c62d1e1ac18e74acf65a
Reviewed-on: http://gerrit.cloudera.org:8080/11066
Reviewed-by: Dan Burkert 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/34b058b8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/34b058b8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/34b058b8

Branch: refs/heads/master
Commit: 34b058b8b89a49e93de61457d67fe9b57f29b0bb
Parents: fb79471
Author: Alex Rodoni 
Authored: Thu Jul 26 14:44:17 2018 -0700
Committer: Dan Burkert 
Committed: Thu Jul 26 22:13:55 2018 +

--
 docs/administration.adoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/34b058b8/docs/administration.adoc
--
diff --git a/docs/administration.adoc b/docs/administration.adoc
index baea3af..e5902f3 100644
--- a/docs/administration.adoc
+++ b/docs/administration.adoc
@@ -762,7 +762,7 @@ removal of a directory across which tablets are configured 
to spread data. If
 `--force` is specified, all tablets configured to use that directory will fail
 upon starting up and be replicated elsewhere.
 
-NOTE: If the link:configuration.html$directory_configuration[metadata
+NOTE: If the link:configuration.html#directory_configuration[metadata
 directory] overlaps with a data directory, as was the default prior to Kudu
 1.7, or if a non-default metadata directory is configured, the
 `--fs_metadata_dir` configuration must be specified when running the `kudu fs

kudu git commit: Add alter_external_catalogs flag to table rename tool

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 141ee5a52 -> d6ee0047c


Add alter_external_catalogs flag to table rename tool

It's going to be necessary to be able to alter the Kudu catalog
independently of the HMS in order to repair and upgrade tables when the
HMS integration is turned on.  A follow-up commit will include a test
that exercises this flag when the HMS integration is enabled.

Change-Id: I9f5d97f234283b7740df727e085523c7f0b09735
Reviewed-on: http://gerrit.cloudera.org:8080/11017
Reviewed-by: Hao Hao 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d6ee0047
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d6ee0047
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d6ee0047

Branch: refs/heads/master
Commit: d6ee0047cef6fc5e3777045d50b1cfe861f745e9
Parents: 141ee5a
Author: Dan Burkert 
Authored: Fri Jul 20 14:07:32 2018 -0700
Committer: Dan Burkert 
Committed: Tue Jul 24 23:54:18 2018 +

--
 src/kudu/client/client.h | 15 ---
 src/kudu/tools/kudu-tool-test.cc | 14 +-
 src/kudu/tools/tool_action_common.cc |  4 
 src/kudu/tools/tool_action_common.h  |  4 
 src/kudu/tools/tool_action_hms.cc|  2 +-
 src/kudu/tools/tool_action_table.cc  |  5 +
 6 files changed, 27 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d6ee0047/src/kudu/client/client.h
--
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 13e25da..cbf3257 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -58,19 +58,17 @@ class PartitionSchema;
 class SecurityUnknownTskTest;
 
 namespace client {
-class KuduClient;
+class KuduTableAlterer;
 }
 
 namespace tools {
 class LeaderMasterProxy;
-
-Status AlterKuduTableOnly(client::KuduClient* kudu_client,
-  const std::string& name,
-  const std::string& new_name);
+void SetAlterExternalCatalogs(client::KuduTableAlterer*, bool);
 } // namespace tools
 
 namespace client {
 
+class KuduClient;
 class KuduDelete;
 class KuduInsert;
 class KuduLoggingCallback;
@@ -79,7 +77,6 @@ class KuduScanBatch;
 class KuduSession;
 class KuduStatusCallback;
 class KuduTable;
-class KuduTableAlterer;
 class KuduTableCreator;
 class KuduTablet;
 class KuduTabletServer;
@@ -1193,11 +1190,7 @@ class KUDU_EXPORT KuduTableAlterer {
 
   friend class KuduClient;
 
-  friend Status tools::AlterKuduTableOnly(
-  client::KuduClient* kudu_client,
-  const std::string& name,
-  const std::string& new_name);
-
+  friend void tools::SetAlterExternalCatalogs(KuduTableAlterer*, bool);
   FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
   FRIEND_TEST(kudu::MasterHmsUpgradeTest, TestRenameExistingTables);
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/d6ee0047/src/kudu/tools/kudu-tool-test.cc
--
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index d19dd15..f9ca66b 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -2013,6 +2013,10 @@ TEST_F(ToolTest, TestMasterList) {
 
 TEST_F(ToolTest, TestRenameTable) {
   NO_FATALS(StartExternalMiniCluster());
+  shared_ptr client;
+  ASSERT_OK(cluster_->CreateClient(nullptr, ));
+  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+
   const string& kTableName = "kudu.table";
   const string& kNewTableName = "kudu_table";
 
@@ -2022,17 +2026,17 @@ TEST_F(ToolTest, TestRenameTable) {
   workload.set_num_replicas(1);
   workload.Setup();
 
-  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
   string out;
   NO_FATALS(RunActionStdoutNone(Substitute("table rename_table $0 $1 $2",
master_addr, kTableName,
kNewTableName)));
-  shared_ptr client;
-  ASSERT_OK(KuduClientBuilder()
-  .add_master_server_addr(master_addr)
-  .Build());
   shared_ptr table;
   ASSERT_OK(client->OpenTable(kNewTableName, ));
+
+  NO_FATALS(RunActionStdoutNone(
+Substitute("table rename_table $0 $1 $2 --noalter_external_catalogs",
+  master_addr, kNewTableName, kTableName)));
+  ASSERT_OK(client->OpenTable(kTableName, ));
 }
 
 TEST_F(ToolTest, TestRenameColumn) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/d6ee0047/src/kudu/tools/tool_action_common.cc
--
diff --git a/src/kudu/tools/tool_action_common.cc 
b/src/kudu/tools/tool_action_common.cc
index e3bcc8f..c0e7301 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ 

kudu git commit: Allow non-HMS compatible tables to be renamed after turning on HMS integration

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master ccdb6b557 -> 7fe04438e


Allow non-HMS compatible tables to be renamed after turning on HMS integration

This is a bug fix followup to cee17c03bc30037bf2a, which introduced
automatic downcasing of table names during new table creation and table
renaming.

Change-Id: If554efbda06e33b5fdb40565e700bad8a306c143
Reviewed-on: http://gerrit.cloudera.org:8080/11016
Reviewed-by: Adar Dembo 
Reviewed-by: Hao Hao 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7fe04438
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7fe04438
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7fe04438

Branch: refs/heads/master
Commit: 7fe04438ecd8e100509ba224826e753ffc851d49
Parents: ccdb6b5
Author: Dan Burkert 
Authored: Tue Jul 17 17:33:00 2018 -0700
Committer: Dan Burkert 
Committed: Tue Jul 24 21:14:07 2018 +

--
 src/kudu/client/client.h   |  2 ++
 src/kudu/integration-tests/master_hms-itest.cc | 29 +++
 src/kudu/master/catalog_manager.cc | 31 -
 3 files changed, 49 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7fe04438/src/kudu/client/client.h
--
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 9ad7afc..13e25da 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -52,6 +52,7 @@ namespace kudu {
 class ClientStressTest_TestUniqueClientIds_Test;
 class KuduPartialRow;
 class MasterHmsTest_TestAlterTable_Test;
+class MasterHmsUpgradeTest_TestRenameExistingTables_Test;
 class MonoDelta;
 class PartitionSchema;
 class SecurityUnknownTskTest;
@@ -1198,6 +1199,7 @@ class KUDU_EXPORT KuduTableAlterer {
   const std::string& new_name);
 
   FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
+  FRIEND_TEST(kudu::MasterHmsUpgradeTest, TestRenameExistingTables);
 
   KuduTableAlterer(KuduClient* client,
const std::string& name);

http://git-wip-us.apache.org/repos/asf/kudu/blob/7fe04438/src/kudu/integration-tests/master_hms-itest.cc
--
diff --git a/src/kudu/integration-tests/master_hms-itest.cc 
b/src/kudu/integration-tests/master_hms-itest.cc
index 4706a9d..7ba3c8c 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -579,4 +579,33 @@ TEST_F(MasterHmsUpgradeTest, 
TestConflictingNormalizedNames) {
   std::sort(tables.begin(), tables.end());
   ASSERT_EQ(tables, vector({ "default.MyTable", 
"default.mytable-renamed" }));
 }
+
+// Checks that existing tables with HMS-incompatible names can be renamed post
+// upgrade using a Kudu-catalog only alter.
+TEST_F(MasterHmsUpgradeTest, TestRenameExistingTables) {
+  ASSERT_OK(CreateKuduTable("default", "UPPERCASE"));
+  ASSERT_OK(CreateKuduTable("default", "illegal-chars⁉"));
+
+  // Shutdown the masters and turn on the HMS integration
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
+  cluster_->EnableMetastoreIntegration();
+  ASSERT_OK(cluster_->Restart());
+
+  vector tables;
+  ASSERT_OK(client_->ListTables());
+  std::sort(tables.begin(), tables.end());
+  ASSERT_EQ(tables, vector({ "default.UPPERCASE", 
"default.illegal-chars⁉" }));
+
+  // Rename the tables using a Kudu catalog only rename.
+  unique_ptr 
alterer(client_->NewTableAlterer("default.UPPERCASE"));
+  
ASSERT_OK(alterer->RenameTo("default.uppercase")->alter_external_catalogs(false)->Alter());
+
+  alterer.reset(client_->NewTableAlterer("default.illegal-chars⁉"));
+  
ASSERT_OK(alterer->RenameTo("default.illegal_chars")->alter_external_catalogs(false)->Alter());
+
+  tables.clear();
+  client_->ListTables();
+  std::sort(tables.begin(), tables.end());
+  ASSERT_EQ(tables, vector({ "default.illegal_chars", 
"default.uppercase" }));
+}
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/7fe04438/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index b10a4cc..d6b4654 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -2261,7 +2261,7 @@ Status CatalogManager::AlterTable(const 
AlterTableRequestPB& req,
 resp, MasterErrorPB::TABLE_NOT_FOUND);
   }
 
-  string table_name = l.data().name();
+  string normalized_table_name = NormalizeTableName(l.data().name());
   *resp->mutable_table_id() = table->id();
 
   // 3. Calculate and validate new schema for the on-disk state, not persisted 
yet.
@@ -2288,22 +2288,27 @@ Status 

kudu git commit: hms-tool: replace a few copies with const refs

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master a68c9355b -> ec1e47f41


hms-tool: replace a few copies with const refs

This patch contains no functional changes.

Change-Id: Id2ace8469b82c995f6f528aff4322bf4fed22bd2
Reviewed-on: http://gerrit.cloudera.org:8080/11015
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ec1e47f4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ec1e47f4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ec1e47f4

Branch: refs/heads/master
Commit: ec1e47f41f25d926609e395d77d0009e210ac17d
Parents: a68c935
Author: Dan Burkert 
Authored: Thu Jul 5 10:06:53 2018 -0700
Committer: Dan Burkert 
Committed: Mon Jul 23 22:54:28 2018 +

--
 src/kudu/tools/tool_action_hms.cc | 36 +++---
 1 file changed, 20 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ec1e47f4/src/kudu/tools/tool_action_hms.cc
--
diff --git a/src/kudu/tools/tool_action_hms.cc 
b/src/kudu/tools/tool_action_hms.cc
index 70a1aeb..751cf3c 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -97,13 +97,12 @@ unordered_map 
RetrieveTablesMap(vector hms_tab
   return hms_tables_map;
 }
 
-string RenameHiveIncompatibleTable(const string& name) {
-  string table_name(name);
+string RenameHiveIncompatibleTable(const string& table_name) {
   cout << Substitute("Table $0 is not hive compatible.", table_name) << endl;
   cout << "Please input a new table name: ";
-  getline(cin, table_name);
-
-  return table_name;
+  string new_table_name;
+  getline(cin, new_table_name);
+  return new_table_name;
 }
 
 // Only alter the table in Kudu but not in the Hive Metastore.
@@ -285,8 +284,7 @@ Status HmsDowngrade(const RunnerContext& context) {
 
   // 2. Downgrades all Kudu tables to legacy table format.
   for (auto& hms_table : hms_tables) {
-if (hms_table.parameters[HmsClient::kStorageHandlerKey] ==
-HmsClient::kKuduStorageHandler) {
+if (hms_table.parameters[HmsClient::kStorageHandlerKey] == 
HmsClient::kKuduStorageHandler) {
   RETURN_NOT_OK(hms_catalog->DowngradeToLegacyImpalaTable(
   Substitute("$0.$1", hms_table.dbName, hms_table.tableName)));
 }
@@ -338,16 +336,22 @@ Status PrintUnsyncedTables(const string& master_addresses,
   DataTable table({ "TableID", "KuduTableName", "HmsDbName", "HmsTableName",
 "KuduMasterAddresses", "HmsTableMasterAddresses"});
   for (const auto& entry : tables_map) {
-string table_id = entry.first;
-shared_ptr kudu_table = entry.second.first;
-vector hms_tables = entry.second.second;
-string kudu_table_name = kudu_table->name();
+const string& table_id = entry.first;
+const KuduTable& kudu_table = *entry.second.first.get();
+const vector& hms_tables = entry.second.second;
+const string& kudu_table_name = kudu_table.name();
 if (hms_tables.empty()) {
   table.AddRow({ table_id, kudu_table_name, "", "", master_addresses, "" 
});
 } else {
-  for (hive::Table hms_table : hms_tables) {
-table.AddRow({table_id, kudu_table_name, hms_table.dbName, 
hms_table.tableName,
- master_addresses, 
hms_table.parameters[HmsClient::kKuduMasterAddrsKey]});
+  for (const hive::Table& hms_table : hms_tables) {
+table.AddRow({
+table_id,
+kudu_table_name,
+hms_table.dbName,
+hms_table.tableName,
+master_addresses,
+FindOrDie(hms_table.parameters, HmsClient::kKuduMasterAddrsKey),
+});
   }
 }
   }
@@ -362,8 +366,8 @@ Status PrintLegacyTables(const vector& tables, 
ostream& out) {
   DataTable table({ "HmsDbName", "HmsTableName", "KuduTableName",
 "KuduMasterAddresses"});
   for (hive::Table t : tables) {
-string kudu_table_name = t.parameters[HmsClient::kLegacyKuduTableNameKey];
-string master_addresses = t.parameters[HmsClient::kKuduMasterAddrsKey];
+const string& kudu_table_name = 
t.parameters[HmsClient::kLegacyKuduTableNameKey];
+const string& master_addresses = 
t.parameters[HmsClient::kKuduMasterAddrsKey];
 table.AddRow({ t.dbName, t.tableName, kudu_table_name, master_addresses });
   }
   return table.PrintTo(out);

[2/2] kudu git commit: [DOCS] Updated the max and recommended numbers of tablets per server

[danburkert]

[DOCS] Updated the max and recommended numbers of tablets per server

Change-Id: Ida16e45477bd18206f25db8cc2bf890742d4ec63
Reviewed-on: http://gerrit.cloudera.org:8080/10969
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/52118974
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/52118974
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/52118974

Branch: refs/heads/master
Commit: 521189744471b7f8c4168a814f64943c776a82b1
Parents: cb03085
Author: Alex Rodoni 
Authored: Tue Jul 17 17:46:34 2018 -0700
Committer: Dan Burkert 
Committed: Wed Jul 18 22:46:09 2018 +

--
 docs/known_issues.adoc | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/52118974/docs/known_issues.adoc
--
diff --git a/docs/known_issues.adoc b/docs/known_issues.adoc
index 1171ca1..687266e 100644
--- a/docs/known_issues.adoc
+++ b/docs/known_issues.adoc
@@ -136,7 +136,8 @@
 * Recommended maximum amount of stored data, post-replication and 
post-compression,
   per tablet server is 8TB.
 
-* Recommended maximum number of tablets per tablet server is 2000, 
post-replication.
+* The maximum number of tablets per tablet server is 2000, post-replication,
+  but we recommend 1000 tablets or fewer per tablet server.
 
 * Maximum number of tablets per table for each tablet server is 60, 
post-replication,
   at table-creation time.
@@ -176,6 +177,7 @@ to communicate only the most important known issues.
   and clients will experience frequent timeouts, and the cluster may become 
unusable.
 
 * If a tablet server has a very large number of tablets, it may take several 
minutes
-  to start up. It is recommended to limit the number of tablets per server to 
100 or fewer.
+  to start up. It is recommended to limit the number of tablets per server to 
1000
+  or fewer. The maximum allowed number of tablets per server is 2000.
   Consider this limitation when pre-splitting your tables. If you notice slow 
start-up times,
   you can monitor the number of tablets per server in the web UI.

[1/2] kudu git commit: [DOCS] Removed KUDU-1942 and KUDU-2032 from Known Issues List

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master c2cecb005 -> 521189744


[DOCS] Removed KUDU-1942 and KUDU-2032 from Known Issues List

Change-Id: I84bdf0193df08241ea632e1b1b51476bba40403c
Reviewed-on: http://gerrit.cloudera.org:8080/10953
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/cb03085b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/cb03085b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/cb03085b

Branch: refs/heads/master
Commit: cb03085b355a02586f0f4a808e32f5c21b92b532
Parents: c2cecb0
Author: Alex Rodoni 
Authored: Mon Jul 16 10:46:50 2018 -0700
Committer: Dan Burkert 
Committed: Wed Jul 18 22:46:06 2018 +

--
 docs/known_issues.adoc | 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/cb03085b/docs/known_issues.adoc
--
diff --git a/docs/known_issues.adoc b/docs/known_issues.adoc
index 1bfb4e7..1171ca1 100644
--- a/docs/known_issues.adoc
+++ b/docs/known_issues.adoc
@@ -179,9 +179,3 @@ to communicate only the most important known issues.
   to start up. It is recommended to limit the number of tablets per server to 
100 or fewer.
   Consider this limitation when pre-splitting your tables. If you notice slow 
start-up times,
   you can monitor the number of tablets per server in the web UI.
-
-* Kerberos authentication does not function correctly on hosts which contain
-  capital letters in their hostname.
-
-* Kerberos authentication does not function correctly if `rdns = false` is 
configured
-  in `krb5.conf`.

kudu git commit: KUDU-2312: Scan predicate application ordering is non-deterministic

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.6.x f0d95c27e -> f43c77bed


KUDU-2312: Scan predicate application ordering is non-deterministic

This changes the scan predicate evaluation ordering so that it primarily
orders by selectivity (as before), but breaks ties by column index.

Change-Id: I99b2cabecd8626cad7e11fbdd492af7276e08348
Reviewed-on: http://gerrit.cloudera.org:8080/9440
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 
Reviewed-on: http://gerrit.cloudera.org:8080/10945


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f43c77be
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f43c77be
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f43c77be

Branch: refs/heads/branch-1.6.x
Commit: f43c77bed464556b66bf8f035dc50be541d6cdb4
Parents: f0d95c2
Author: Dan Burkert 
Authored: Tue Feb 20 11:17:48 2018 -0800
Committer: Dan Burkert 
Committed: Tue Jul 17 22:10:48 2018 +

--
 src/kudu/common/generic_iterators-test.cc | 82 +-
 src/kudu/common/generic_iterators.cc  | 34 +++
 src/kudu/common/generic_iterators.h   | 14 +++--
 src/kudu/tablet/cfile_set.h   |  1 -
 4 files changed, 111 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f43c77be/src/kudu/common/generic_iterators-test.cc
--
diff --git a/src/kudu/common/generic_iterators-test.cc 
b/src/kudu/common/generic_iterators-test.cc
index e7f9250..55eaa09 100644
--- a/src/kudu/common/generic_iterators-test.cc
+++ b/src/kudu/common/generic_iterators-test.cc
@@ -51,6 +51,7 @@ DEFINE_int32(num_lists, 3, "Number of lists to merge");
 DEFINE_int32(num_rows, 1000, "Number of entries per list");
 DEFINE_int32(num_iters, 1, "Number of times to run merge");
 
+using std::make_shared;
 using std::shared_ptr;
 using std::string;
 using std::vector;
@@ -309,7 +310,7 @@ TEST(TestPredicateEvaluatingIterator, 
TestPredicateEvaluation) {
 
   ASSERT_EQ(0, spec.predicates().size())
 << "Iterator tree should have accepted predicate";
-  ASSERT_EQ(1, pred_eval->col_idx_predicates_.size())
+  ASSERT_EQ(1, pred_eval->col_predicates_.size())
 << "Predicate should be evaluated by the outer iterator";
 
   Arena arena(1024);
@@ -338,4 +339,83 @@ TEST(TestPredicateEvaluatingIterator, 
TestDontWrapWhenNoPredicates) {
   ASSERT_EQ(outer_iter, materializing) << "InitAndMaybeWrap should not have 
wrapped iter";
 }
 
+// Test row-wise iterator which does nothing.
+class DummyIterator : public RowwiseIterator {
+ public:
+
+  explicit DummyIterator(Schema schema)
+  : schema_(std::move(schema)) {
+  }
+
+  Status Init(ScanSpec* /*spec*/) override {
+return Status::OK();
+  }
+
+  virtual Status NextBlock(RowBlock* /*dst*/) override {
+LOG(FATAL) << "unimplemented!";
+return Status::OK();
+  }
+
+  virtual bool HasNext() const override {
+LOG(FATAL) << "unimplemented!";
+return false;
+  }
+
+  virtual string ToString() const override {
+return "DummyIterator";
+  }
+
+  virtual const Schema& schema() const override {
+return schema_;
+  }
+
+  virtual void GetIteratorStats(vector* stats) const override {
+stats->resize(schema().num_columns());
+  }
+
+ private:
+  Schema schema_;
+};
+
+TEST(TestPredicateEvaluatingIterator, TestPredicateEvaluationOrder) {
+  Schema schema({ ColumnSchema("a_int64", INT64),
+  ColumnSchema("b_int64", INT64),
+  ColumnSchema("c_int32", INT32) }, 3);
+
+  int64_t zero = 0;
+  int64_t two = 2;
+  auto a_equality = ColumnPredicate::Equality(schema.column(0), );
+  auto b_equality = ColumnPredicate::Equality(schema.column(1), );
+  auto c_equality = ColumnPredicate::Equality(schema.column(2), );
+  auto a_range = ColumnPredicate::Range(schema.column(0), , );
+
+  { // Test that more selective predicates come before others.
+ScanSpec spec;
+spec.AddPredicate(a_range);
+spec.AddPredicate(b_equality);
+spec.AddPredicate(c_equality);
+
+shared_ptr iter = make_shared(schema);
+ASSERT_OK(PredicateEvaluatingIterator::InitAndMaybeWrap(, ));
+
+PredicateEvaluatingIterator* pred_eval = 
down_cast(iter.get());
+ASSERT_TRUE(pred_eval->col_predicates_ ==
+vector({ c_equality, b_equality, a_range }));
+  }
+
+  { // Test that smaller columns come before larger ones, and ties are broken 
by idx.
+ScanSpec spec;
+spec.AddPredicate(b_equality);
+spec.AddPredicate(a_equality);
+spec.AddPredicate(c_equality);
+
+shared_ptr iter = make_shared(schema);
+ASSERT_OK(PredicateEvaluatingIterator::InitAndMaybeWrap(, ));
+
+PredicateEvaluatingIterator* pred_eval = 
down_cast(iter.get());
+ASSERT_TRUE(pred_eval->col_predicates_ ==
+

kudu git commit: KUDU-2447 Fix TS crashed with "NONE predicate can not be pushed into key"

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.7.x a307f2a31 -> 2a8893e3e


KUDU-2447 Fix TS crashed with "NONE predicate can not be pushed into key"

tserver crashes on Scan when [lowerPrimaryKey, upperPrimary)
 and predicates of primary column do not overlap

Change-Id: I02810d651e14768f0a85044b0b7f01fa7246c718
Reviewed-on: http://gerrit.cloudera.org:8080/10514
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 
(cherry picked from commit c7918e87bf1aeb25d5b08f7b340d1ae15806568d)
Reviewed-on: http://gerrit.cloudera.org:8080/10963
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2a8893e3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2a8893e3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2a8893e3

Branch: refs/heads/branch-1.7.x
Commit: 2a8893e3e9aed3e36f21bb74baafc93f713b592e
Parents: a307f2a
Author: oclarms 
Authored: Fri May 25 19:55:36 2018 +0800
Committer: Dan Burkert 
Committed: Tue Jul 17 21:49:22 2018 +

--
 .../kudu/client/TestScannerMultiTablet.java |  7 +-
 src/kudu/common/scan_spec-test.cc   | 81 ++--
 src/kudu/common/scan_spec.cc|  5 +-
 3 files changed, 67 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/2a8893e3/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
index 22a5bed..1d4581f 100644
--- 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
+++ 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
@@ -121,8 +121,13 @@ public class TestScannerMultiTablet extends BaseKuduTest {
   // Test mixing start/end row keys with predicates.
   @Test(timeout = 10)
   public void testKeysAndPredicates() throws Exception {
+// Value that doesn't exist, predicates has primary column
+ColumnRangePredicate predicate = new 
ColumnRangePredicate(schema.getColumnByIndex(1));
+predicate.setUpperBound("1");
+assertEquals(0, countRowsInScan(getScanner("1", "2", "1", "3", 
predicate)));
+
 // First row from the 2nd tablet.
-ColumnRangePredicate predicate = new 
ColumnRangePredicate(schema.getColumnByIndex(2));
+predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
 predicate.setLowerBound("1");
 predicate.setUpperBound("1");
 assertEquals(1, countRowsInScan(getScanner("1", "", "2", "", predicate)));

http://git-wip-us.apache.org/repos/asf/kudu/blob/2a8893e3/src/kudu/common/scan_spec-test.cc
--
diff --git a/src/kudu/common/scan_spec-test.cc 
b/src/kudu/common/scan_spec-test.cc
index e24f9d2..d925510 100644
--- a/src/kudu/common/scan_spec-test.cc
+++ b/src/kudu/common/scan_spec-test.cc
@@ -643,34 +643,67 @@ TEST_F(CompositeIntKeysTest, 
TestLiftPrimaryKeyBounds_BothBounds) {
 // Test that implicit constraints specified in the primary key upper/lower
 // bounds are merged into the set of predicates.
 TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_WithPredicates) {
-  // b >= 15
-  // c >= 3
-  // c <= 100
-  // key >= (10, min, min)
-  //  < (10,  90, min)
-  ScanSpec spec;
-  AddPredicate(, "b", GE, 15);
-  AddPredicate(, "c", GE, 3);
-  AddPredicate(, "c", LE, 100);
+  {
+// b >= 15
+// c >= 3
+// c <= 100
+// key >= (10, min, min)
+//  < (10,  90, min)
+ScanSpec spec;
+AddPredicate(, "b", GE, 15);
+AddPredicate(, "c", GE, 3);
+AddPredicate(, "c", LE, 100);
 
-  KuduPartialRow lower_bound(_);
-  CHECK_OK(lower_bound.SetInt8("a", 10));
-  CHECK_OK(lower_bound.SetInt8("b", INT8_MIN));
-  CHECK_OK(lower_bound.SetInt8("c", INT8_MIN));
+KuduPartialRow lower_bound(_);
+CHECK_OK(lower_bound.SetInt8("a", 10));
+CHECK_OK(lower_bound.SetInt8("b", INT8_MIN));
+CHECK_OK(lower_bound.SetInt8("c", INT8_MIN));
 
-  KuduPartialRow upper_bound(_);
-  CHECK_OK(upper_bound.SetInt8("a", 10));
-  CHECK_OK(upper_bound.SetInt8("b", 90));
-  CHECK_OK(upper_bound.SetInt8("c", INT8_MIN));
+KuduPartialRow upper_bound(_);
+CHECK_OK(upper_bound.SetInt8("a", 10));
+CHECK_OK(upper_bound.SetInt8("b", 90));
+CHECK_OK(upper_bound.SetInt8("c", INT8_MIN));
 
-  SetLowerBound(, lower_bound);
-  SetExclusiveUpperBound(, upper_bound);
+SetLowerBound(, lower_bound);
+SetExclusiveUpperBound(, upper_bound);
 
-  spec.OptimizeScan(schema_, _, _, false);
-  ASSERT_EQ(3, spec.predicates().size());
-  ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), 

kudu git commit: hms-tool: filter non-Kudu tables in the HMS

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master f62bbdd64 -> d074de85d


hms-tool: filter non-Kudu tables in the HMS

Renames HmsCatalog::RetrieveTables to GetKuduTables and changes the
semantics such that only Kudu tables are returned. The only caller was
the HMS tool, and it only needs to inspect Kudu tables.

Special APIs are added to the HMS client and HMS catalog layers that
allow sending a filter to the HMS to strip out non-Kudu tables when
listing tables, as well as a bulk-get API. The combination of these APIs
should be significantly more efficient than issuing a get for every
single table in the HMS and doing Kudu-side filtering.

Also included are some style and formatting fixups in HMS tool.

This patch includes no functional changes to the HMS tool.

Change-Id: I5f83d2e705ea6910a9aa0a1eda0d30b5feb2607b
Reviewed-on: http://gerrit.cloudera.org:8080/10934
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d074de85
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d074de85
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d074de85

Branch: refs/heads/master
Commit: d074de85dd19697444bb48eabe5b5c577a579d07
Parents: f62bbdd
Author: Dan Burkert 
Authored: Tue Jul 3 17:12:59 2018 -0700
Committer: Dan Burkert 
Committed: Fri Jul 13 02:34:34 2018 +

--
 src/kudu/hms/hms_catalog-test.cc| 37 +-
 src/kudu/hms/hms_catalog.cc | 27 +--
 src/kudu/hms/hms_catalog.h  | 12 ++-
 src/kudu/hms/hms_client-test.cc | 49 ++---
 src/kudu/hms/hms_client.cc  | 30 +++-
 src/kudu/hms/hms_client.h   | 19 -
 src/kudu/integration-tests/master_hms-itest.cc  |  2 +-
 .../mini-cluster/external_mini_cluster-test.cc  |  2 +-
 src/kudu/tools/kudu-tool-test.cc|  6 +-
 src/kudu/tools/tool_action_hms.cc   | 77 +---
 10 files changed, 167 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d074de85/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 472f3a8..4574402 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -396,36 +396,39 @@ TEST_F(HmsCatalogTest, TestExternalTable) {
   NO_FATALS(CheckTableDoesNotExist("default", "bogus_table_name"));
 }
 
-TEST_F(HmsCatalogTest, TestRetrieveTables) {
+TEST_F(HmsCatalogTest, TestGetKuduTables) {
   const string kHmsDatabase = "db";
   const string kManagedTableName = "managed_table";
   const string kExternalTableName = "external_table";
+  const string kTableName = "external_table";
   const string kNonKuduTableName = "non_kudu_table";
 
-  // Create a Impala managed table, a external table and a non Kudu table.
+  // Create a legacy Impala managed table, a legacy Impala external table, a
+  // Kudu table, and a non Kudu table.
   hive::Database db;
-  db.name = kHmsDatabase;
+  db.name = "db";
   ASSERT_OK(hms_client_->CreateDatabase(db));
-  ASSERT_OK(CreateLegacyTable(kHmsDatabase,
-  kManagedTableName,
-  HmsClient::kManagedTable));
+  ASSERT_OK(CreateLegacyTable("db", "managed_table", 
HmsClient::kManagedTable));
   hive::Table table;
-  ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kManagedTableName, ));
-  ASSERT_OK(CreateLegacyTable(kHmsDatabase,
-  kExternalTableName,
-  HmsClient::kExternalTable));
-  ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kExternalTableName, ));
+  ASSERT_OK(hms_client_->GetTable(kHmsDatabase, "managed_table", ));
+  ASSERT_OK(CreateLegacyTable("db", "external_table", 
HmsClient::kExternalTable));
+  ASSERT_OK(hms_client_->GetTable("db", "external_table", ));
+
+  ASSERT_OK(hms_catalog_->CreateTable("fake-id", "db.table", Schema()));
 
   hive::Table non_kudu_table;
-  non_kudu_table.dbName = kHmsDatabase;
-  non_kudu_table.tableName = kNonKuduTableName;
+  non_kudu_table.dbName = "db";
+  non_kudu_table.tableName = "non_kudu_table";
   ASSERT_OK(hms_client_->CreateTable(non_kudu_table));
-  ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kNonKuduTableName, ));
+  ASSERT_OK(hms_client_->GetTable("db", "non_kudu_table", ));
 
   // Retrieve all tables and ensure all entries are found.
-  vector hms_tables;
-  ASSERT_OK(hms_catalog_->RetrieveTables(_tables));
-  ASSERT_EQ(3, hms_tables.size());
+  vector kudu_tables;
+  ASSERT_OK(hms_catalog_->GetKuduTables(_tables));
+  ASSERT_EQ(3, kudu_tables.size());
+  for (const auto& kudu_table : kudu_tables) {
+ASSERT_FALSE(kudu_table.tableName == 

kudu git commit: KUDU-2191: downcase/normalize table names during DDL

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 4eefc8d4e -> cee17c03b


KUDU-2191: downcase/normalize table names during DDL

This is a followup to 7b048b8dbe which changed the catalog manager to be
case preserving, but insensitive on lookup when the HMS integration is
enabled. It turns out this was only possible because the HMS failed to
downcase/normalize table names in notification log events[1]. This is an
oversight, and probably could be considered a bug, and HMS developers
have suggested that Kudu should not rely on it.

After a lot of consideration I haven't been able to come up with a way
to keep the case preserving semantics without changes to the HMS APIs,
so instead this commit throws in the towel and adopts HMS-style case
normalization during CREATE TABLE and ALTER TABLE RENAME operations.

Existing tables with uppercase table names will not be altered
automatically (this is consistent with the current handling of non-ascii
chars in table names), so the upgrade CLI tool will be extended in a
follow up commit to handle this.

[1] In particular, renaming a table is problematic if the notification
log listener events doesn't preserve case due to how the catalog manager
/ notification log listener handles table renames.

Change-Id: Ie32a209d9d85851562691ddbc30f7dd02886bad7
Reviewed-on: http://gerrit.cloudera.org:8080/10903
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/cee17c03
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/cee17c03
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/cee17c03

Branch: refs/heads/master
Commit: cee17c03bc30037bf2a7d97c6bb3aa39cec34a4c
Parents: 4eefc8d
Author: Dan Burkert 
Authored: Tue Jul 10 11:07:03 2018 -0700
Committer: Dan Burkert 
Committed: Wed Jul 11 23:01:51 2018 +

--
 src/kudu/integration-tests/master_hms-itest.cc | 19 ++---
 src/kudu/master/catalog_manager.cc | 91 +
 2 files changed, 46 insertions(+), 64 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/cee17c03/src/kudu/integration-tests/master_hms-itest.cc
--
diff --git a/src/kudu/integration-tests/master_hms-itest.cc 
b/src/kudu/integration-tests/master_hms-itest.cc
index 4b639f5..2fd415c 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -499,23 +499,16 @@ TEST_F(MasterHmsTest, TestUppercaseIdentifiers) {
 ASSERT_EQ(name, table->name());
   }
 
-  // Listing tables shows the preserved case.
+  // Listing tables shows the normalized case.
   vector tables;
   ASSERT_OK(client_->ListTables());
-  ASSERT_EQ(tables, vector({ "default.MyTable" }));
+  ASSERT_EQ(tables, vector({ "default.mytable" }));
 
   // Rename the table to the same normalized name, but with a different case.
   unique_ptr table_alterer;
   table_alterer.reset(client_->NewTableAlterer("default.mytable"));
-  ASSERT_OK(table_alterer->RenameTo("DEFAULT.MYTABLE")->Alter());
-  NO_FATALS(CheckTable("default", "MyTable"));
-  NO_FATALS(CheckTable("default", "mytable"));
-  NO_FATALS(CheckTable("default", "MYTABLE"));
-
-  // The master should retain the new case.
-  tables.clear();
-  ASSERT_OK(client_->ListTables());
-  ASSERT_EQ(tables, vector({ "DEFAULT.MYTABLE" }));
+  Status s = table_alterer->RenameTo("DEFAULT.MYTABLE")->Alter();
+  ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString();
 
   // Rename the table to something different.
   table_alterer.reset(client_->NewTableAlterer("DEFAULT.MYTABLE"));
@@ -530,10 +523,10 @@ TEST_F(MasterHmsTest, TestUppercaseIdentifiers) {
 NO_FATALS(CheckTable("default", "AbC"));
   });
 
-  // Listing tables shows the preserved case.
+  // Listing tables shows the normalized case.
   tables.clear();
   ASSERT_OK(client_->ListTables());
-  ASSERT_EQ(tables, vector({ "default.AbC" }));
+  ASSERT_EQ(tables, vector({ "default.abc" }));
 
   // Drop the table.
   ASSERT_OK(client_->DeleteTable("DEFAULT.abc"));

http://git-wip-us.apache.org/repos/asf/kudu/blob/cee17c03/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 8f58924..b10a4cc 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -1363,10 +1363,9 @@ Status CatalogManager::CreateTable(const 
CreateTableRequestPB* orig_req,
   // a. Validate the user request.
   Schema client_schema;
   RETURN_NOT_OK(SchemaFromPB(req.schema(), _schema));
-  const string& table_name = req.name();
-  string normalized_table_name = NormalizeTableName(table_name);
+  string normalized_table_name = NormalizeTableName(req.name());
 

kudu git commit: KUDU-2492: Make the use of SO_REUSEPORT conditional on it being defined

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 8c744afa9 -> af8bb879c


KUDU-2492: Make the use of SO_REUSEPORT conditional on it being defined

A recent commit to Kudu, "rpc: add experimental rpc_reuseport flag",
added the use of the rpc flag SO_REUSEREPORT. This flag is not
available with older versions of Linux, resulting in a compiler error.

This patch avoids the compiler error with a macro that checks if
SO_REUSEPORT is defined, and if it's not attempting to set it will
return an error.

Change-Id: I673fa09619955de162e2e2742480672b1e5c4d1d
Reviewed-on: http://gerrit.cloudera.org:8080/10875
Reviewed-by: Dan Burkert 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/af8bb879
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/af8bb879
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/af8bb879

Branch: refs/heads/master
Commit: af8bb879c0a2dd7cd228fad055cd20e14f783842
Parents: 8c744af
Author: Thomas Tauber-Marshall 
Authored: Thu Jul 5 11:45:00 2018 -0700
Committer: Dan Burkert 
Committed: Fri Jul 6 23:09:27 2018 +

--
 src/kudu/util/net/socket.cc | 12 
 1 file changed, 8 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/af8bb879/src/kudu/util/net/socket.cc
--
diff --git a/src/kudu/util/net/socket.cc b/src/kudu/util/net/socket.cc
index cc14702..bc3b525 100644
--- a/src/kudu/util/net/socket.cc
+++ b/src/kudu/util/net/socket.cc
@@ -245,10 +245,14 @@ Status Socket::SetReuseAddr(bool flag) {
 }
 
 Status Socket::SetReusePort(bool flag) {
-  int int_flag = flag ? 1 : 0;
-  RETURN_NOT_OK_PREPEND(SetSockOpt(SOL_SOCKET, SO_REUSEPORT, int_flag),
-"failed to set SO_REUSEPORT");
-  return Status::OK();
+  #ifdef SO_REUSEPORT
+int int_flag = flag ? 1 : 0;
+RETURN_NOT_OK_PREPEND(SetSockOpt(SOL_SOCKET, SO_REUSEPORT, int_flag),
+  "failed to set SO_REUSEPORT");
+return Status::OK();
+  #else
+return Status::NotSupported("failed to set SO_REUSEPORT: protocol not 
available");
+  #endif
 }
 
 Status Socket::BindAndListen(const Sockaddr ,

kudu git commit: KUDU-2191: support table-name identifiers with upper case chars

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 1ae050e4d -> 7b048b8db


KUDU-2191: support table-name identifiers with upper case chars

Summary: When the HMS integration is enabled, Kudu now preserves table
name casing, but uses case-insensitive lookups to retrieve tables.

Background: The HMS lowercases all database (table) identifiers during database
(table) creation, only storing the lowercased version. On database and
table lookup the HMS automatically does a case-insensitive compare.
During table creation Kudu checks that table names are valid UTF-8, and
does no transformations on identiers. During table lookups Kudu requires
that the table name match exactly, including case.

As a result of these behavior differences and the design of the
notification log listener, tables with upper-case characters can not be
altered or deleted when the HMS integration is enabled. This commit
fixes this by changing how the Catalog Manager handles identifiers when
the HMS integration is enabled:

* During table creation, the Catalog Manager preserves the case of table
  names.
* On table lookup, the Catalog Manager does a case-insensitive
  comparison to find the table.

This is implemented by storing the preserved case in the table's
sys-catalog metadata entry, and storing a 'normalized' (down-cased)
identifier in the ephemeral by-name table map. The various parts of the
catalog manager which deal with the by-name map are converted to use the
normalized version of the name. When the HMS integration is not
configured, normalized table names are equal to the original table name,
so the behavior changes that this patch introduces are entirely opt-in.

There is one edge case that complicates turning on the HMS integration
in rare circumstances: if there are existing (legacy) tables with names
which map to the same normalized form (e.g. differ only in case), the
catalog manager will fail to startup and instruct the operator to rename
the offending tables before trying again. Additionally, this check only
applies to tables that otherwise follow the Hive table naming rules
(matching regex '[\w_/]+\.[\w_/]+').

Change-Id: I18977d6fe7b2999a36681a728ac0d1e54b7f38cd
Reviewed-on: http://gerrit.cloudera.org:8080/10817
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7b048b8d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7b048b8d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7b048b8d

Branch: refs/heads/master
Commit: 7b048b8dbe2562e68dd2b2b7f1a4db83b1ae10bf
Parents: 1ae050e
Author: Dan Burkert 
Authored: Fri Jun 22 14:19:38 2018 -0700
Committer: Dan Burkert 
Committed: Tue Jul 3 23:23:32 2018 +

--
 src/kudu/hms/hms_catalog-test.cc|  62 +---
 src/kudu/hms/hms_catalog.cc |  97 +
 src/kudu/hms/hms_catalog.h  |  26 +++-
 src/kudu/hms/hms_client-test.cc |  28 
 .../integration-tests/master-stress-test.cc |   2 +-
 src/kudu/integration-tests/master_hms-itest.cc  | 126 +++--
 src/kudu/master/catalog_manager.cc  | 141 +--
 src/kudu/master/catalog_manager.h   |  34 +++--
 src/kudu/mini-cluster/external_mini_cluster.cc  |  15 ++
 src/kudu/mini-cluster/external_mini_cluster.h   |   8 +-
 10 files changed, 422 insertions(+), 117 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7b048b8d/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 12066f6..472f3a8 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -37,6 +37,7 @@
 #include "kudu/rpc/sasl_common.h"
 #include "kudu/security/test/mini_kdc.h"
 #include "kudu/util/net/net_util.h"
+#include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
@@ -55,35 +56,64 @@ namespace kudu {
 namespace hms {
 
 TEST(HmsCatalogStaticTest, TestParseTableName) {
-  string db;
-  string tbl;
+  Slice db;
+  Slice tbl;
+  string table;
 
-  EXPECT_OK(HmsCatalog::ParseTableName("foo.bar", , ));
+  table = "foo.bar";
+  ASSERT_OK(HmsCatalog::ParseTableName(table, , ));
   EXPECT_EQ("foo", db);
   EXPECT_EQ("bar", tbl);
 
-  EXPECT_OK(HmsCatalog::ParseTableName("99bottles.my_awesome/table/22", , 
));
+  table = "99bottles.my_awesome/table/22";
+  ASSERT_OK(HmsCatalog::ParseTableName(table, , ));
   EXPECT_EQ("99bottles", db);
   EXPECT_EQ("my_awesome/table/22", tbl);
 
-  
EXPECT_OK(HmsCatalog::ParseTableName("_leading_underscore.trailing_underscore_",
 , ));
+  table = "_leading_underscore.trailing_underscore_";
+  

kudu git commit: KUDU-2191: Adjust upgrade tool to tolerate downgraded metadata

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master f5c802ced -> 95182a6df


KUDU-2191: Adjust upgrade tool to tolerate downgraded metadata

Currently, TestHmsDowngrade keeps on failing as the metadata upgrade
tool does not handle tables being downgraded properly. This issue
didn't surface because commit 214dbc249 didn't rebase. This commit
corrects the upgrade tool to better handle this scenario.

I also looped this test for 500 times, no failures found for
TestHmsDowngrade. Dist-test:
http://dist-test.cloudera.org/job?job_id=hao.hao.1530566672.124129

Change-Id: Ie9a3b154e77db4eda72b9c4b53861f9fe06f09a3
Reviewed-on: http://gerrit.cloudera.org:8080/10844
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/95182a6d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/95182a6d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/95182a6d

Branch: refs/heads/master
Commit: 95182a6dff9da1f268d60aed705a267ed2946df6
Parents: f5c802c
Author: Hao Hao 
Authored: Fri Jun 29 14:12:31 2018 -0700
Committer: Hao Hao 
Committed: Mon Jul 2 22:34:30 2018 +

--
 src/kudu/tools/tool_action_hms.cc | 29 +++--
 1 file changed, 15 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/95182a6d/src/kudu/tools/tool_action_hms.cc
--
diff --git a/src/kudu/tools/tool_action_hms.cc 
b/src/kudu/tools/tool_action_hms.cc
index d0e3c4b..f06367b 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -158,20 +158,21 @@ Status AlterLegacyKuduTables(KuduClient* kudu_client,
   if (hms_table->parameters[HmsClient::kStorageHandlerKey] ==
   HmsClient::kLegacyKuduStorageHandler) {
 string new_table_name = Substitute("$0.$1", hms_table->dbName, 
hms_table->tableName);
-bool exist;
-RETURN_NOT_OK(kudu_client->TableExists(new_table_name, ));
-if (!exist) {
-  // TODO(Hao): Use notification listener to avoid race conditions.
-  s = AlterKuduTableOnly(kudu_client, table_name, 
new_table_name).AndThen([&] {
-return hms_catalog->UpgradeLegacyImpalaTable(kudu_table->id(),
-hms_table->dbName, hms_table->tableName,
-client::SchemaFromKuduSchema(kudu_table->schema()));
-  });
-} else {
-  LOG(WARNING) << Substitute("Failed to upgrade legacy Impala table 
'$0.$1' "
- "(Kudu table name: $2), because a Kudu 
table with "
- "name '$0.$1' already exists'.", 
hms_table->dbName,
-  hms_table->tableName, table_name);
+// Hive-compatible table name implies the table has been upgraded 
previously and
+// then downgraded. In this case, we only upgrade the legacy Impala 
table.
+if (table_name != new_table_name) {
+  s = AlterKuduTableOnly(kudu_client, table_name, new_table_name);
+  if (s.IsAlreadyPresent()) {
+s = s.CloneAndPrepend(Substitute("Failed to upgrade legacy Impala 
table '$0.$1' "
+ "(Kudu table name: $2), because a 
Kudu table with "
+ "name '$0.$1' already exists'.", 
hms_table->dbName,
+ hms_table->tableName, 
table_name));
+  }
+}
+if (s.ok()) {
+  s = hms_catalog->UpgradeLegacyImpalaTable(
+  kudu_table->id(), hms_table->dbName, hms_table->tableName,
+  client::SchemaFromKuduSchema(kudu_table->schema()));
 }
   }
 } else {

[2/2] kudu git commit: Add regression test for HIVE-19569

[danburkert]

Add regression test for HIVE-19569

HIVE-19569 affects some versions of the Hive 3.x line, and it will break
our integration. This additional test case will alert us if our bundeled
Hive version becomes susceptible.

Change-Id: I560c65c9f56ed64fd13897951d2a719abb5c066b
Reviewed-on: http://gerrit.cloudera.org:8080/10832
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/36a76467
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/36a76467
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/36a76467

Branch: refs/heads/master
Commit: 36a764679b950d995919aabfeffa086d391f303e
Parents: 214dbc2
Author: Dan Burkert 
Authored: Tue Jun 26 15:15:54 2018 -0700
Committer: Dan Burkert 
Committed: Thu Jun 28 01:34:54 2018 +

--
 src/kudu/integration-tests/master_hms-itest.cc | 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/36a76467/src/kudu/integration-tests/master_hms-itest.cc
--
diff --git a/src/kudu/integration-tests/master_hms-itest.cc 
b/src/kudu/integration-tests/master_hms-itest.cc
index 84ed562..8a7c4a1 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -325,6 +325,21 @@ TEST_F(MasterHmsTest, TestRenameTable) {
   ASSERT_OK(hms_client_->GetAllTables("db", ));
   std::sort(tables.begin(), tables.end());
   ASSERT_EQ(tables, vector({ "b", "d" })) << tables;
+
+  // Regression test for HIVE-19569
+  // If HIVE-19569 is in effect the rename across databases will result in DROP
+  // TABLE and CREATE TABLE events, which will cause the notification log
+  // listener to drop the table. The alter will succeed (see KUDU-2475), but 
the
+  // subsequent checks will fail, since the table will be deleted.
+  ASSERT_OK(CreateDatabase("db1"));
+  ASSERT_OK(CreateDatabase("db2"));
+  ASSERT_OK(CreateKuduTable("db1", "t1"));
+  NO_FATALS(CheckTable("db1", "t1"));
+  table_alterer.reset(client_->NewTableAlterer("db1.t1"));
+  ASSERT_OK(table_alterer->RenameTo("db2.t2")->Alter());
+  NO_FATALS(CheckTable("db2", "t2"));
+  NO_FATALS(CheckTableDoesNotExist("db1", "t1"));
+  NO_FATALS(CheckTableDoesNotExist("db1", "t2"));
 }
 
 TEST_F(MasterHmsTest, TestAlterTable) {

[2/2] kudu git commit: KUDU-2191: reject alter table rename to same table name

[danburkert]

KUDU-2191: reject alter table rename to same table name

This commit ensures 'ALTER TABLE t RENAME TO t;' operations will fail
validation in the catalog manager when the HMS integration is enabled.
The HMS allows these types of alterations, however Kudu (without HMS
integration) does not, so this extra validation aligns the behavior
between the two configurations. Postgres also does not allow this type
of alter.

Change-Id: I37d895bcc98cbc45a27ff25e97a25436273d79b0
Reviewed-on: http://gerrit.cloudera.org:8080/10831
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/29c66db4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/29c66db4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/29c66db4

Branch: refs/heads/master
Commit: 29c66db48b635b7a1d708c7ddb7e0fcd3c8c33fd
Parents: 72d0981
Author: Dan Burkert 
Authored: Tue Jun 26 14:45:01 2018 -0700
Committer: Dan Burkert 
Committed: Wed Jun 27 20:50:36 2018 +

--
 src/kudu/integration-tests/alter_table-test.cc |  6 ++
 src/kudu/integration-tests/master_hms-itest.cc | 10 --
 src/kudu/master/catalog_manager.cc | 12 
 3 files changed, 26 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/29c66db4/src/kudu/integration-tests/alter_table-test.cc
--
diff --git a/src/kudu/integration-tests/alter_table-test.cc 
b/src/kudu/integration-tests/alter_table-test.cc
index 640f1cb..6bc7929 100644
--- a/src/kudu/integration-tests/alter_table-test.cc
+++ b/src/kudu/integration-tests/alter_table-test.cc
@@ -804,6 +804,12 @@ TEST_F(AlterTableTest, TestRenameTableAndAdd) {
   ASSERT_OK(AddNewI32Column(new_name, "new", 0xdeadbeef));
 }
 
+TEST_F(AlterTableTest, TestRenameToSameName) {
+  unique_ptr 
table_alterer(client_->NewTableAlterer(kTableName));
+  Status s = table_alterer->RenameTo(kTableName)->Alter();
+  ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString();
+}
+
 // Test restarting a tablet server several times after various
 // schema changes.
 // This is a regression test for KUDU-462.

http://git-wip-us.apache.org/repos/asf/kudu/blob/29c66db4/src/kudu/integration-tests/master_hms-itest.cc
--
diff --git a/src/kudu/integration-tests/master_hms-itest.cc 
b/src/kudu/integration-tests/master_hms-itest.cc
index 160121d..84ed562 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -268,9 +268,15 @@ TEST_F(MasterHmsTest, TestRenameTable) {
   external_table.tableName = "b";
   ASSERT_OK(hms_client_->CreateTable(external_table));
 
-  // Attempt to rename the Kudu table to the external table name.
+  // Attempt to rename the Kudu table to the same name.
   unique_ptr table_alterer(client_->NewTableAlterer("db.a"));
-  Status s = table_alterer->RenameTo("db.b")->Alter();
+  Status s = table_alterer->RenameTo("db.a")->Alter();
+  ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString();
+  ASSERT_STR_CONTAINS(s.ToString(), "a already exists");
+
+  // Attempt to rename the Kudu table to the external table name.
+  table_alterer.reset(client_->NewTableAlterer("db.a"));
+  s = table_alterer->RenameTo("db.b")->Alter();
   ASSERT_TRUE(s.IsIllegalState()) << s.ToString();
   ASSERT_STR_CONTAINS(s.ToString(), "b already exists");
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/29c66db4/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 49e368b..7094a70 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -2149,6 +2149,18 @@ Status CatalogManager::AlterTableRpc(const 
AlterTableRequestPB& req,
 RETURN_NOT_OK(FindAndLockTable(req, resp, LockMode::READ, , ));
 RETURN_NOT_OK(CheckIfTableDeletedOrNotRunning(, resp));
 
+// The HMS allows renaming a table to the same name (ALTER TABLE t RENAME 
TO t;),
+// however Kudu does not, so we must validate this case ourselves. This
+// invariant is also checked in CatalogManager::AlterTable, but when the 
HMS
+// integration is enabled that check does not bubble up to the client (it's
+// called by the notification log listener).
+if (l.data().name() == req.new_table_name()) {
+  return SetupError(
+  Status::AlreadyPresent(Substitute("table $0 already exists with id 
$1",
+  req.new_table_name(), table->id())),
+  resp, MasterErrorPB::TABLE_ALREADY_PRESENT);
+}
+
 Schema schema;
 RETURN_NOT_OK(SchemaFromPB(l.data().pb.schema(), ));
 

[1/2] kudu git commit: Add a simple metric for cluster skew

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 9c89c2a9a -> 29c66db48


Add a simple metric for cluster skew

This adds a very simple 'cluster_skew' metric to the master that reports
on the difference in number of replicas between the most and least
loaded tablet servers. This information was already computable from the
tablets_num_* metrics available on all the tablet servers, but this
centralizes it in one place and handles counting the correct tablet
states, so it's much easier to consume. This simple metric should be
useful for operators trying to set up simple alerting schemes based on
cluster balance.

Why not introduce a more comprehensive set of metrics around balance?
Because eventually rebalancing should be tightly integrated with the
master. This metric is just meant as a useful "canary" for when the
rebalancer ought to be run, until a more sophisticated and automated
procedure can be put in place. At that time there will likely be better
metrics exposed to gauge the balance of the cluster and the behavior of
the rebalancer.

I also wrote a quick script to simulate placing replicas on tablet
servers and measure the resulting distribution of skew. The results of
the simulations show skew is almost certainly 6 or less when replica
distribution is determined solely by the current power of two choices
algorithm with a fixed number of tablet servers. This can provide some
guide to operators looking to set a threshold for concerning skew: a
value of e.g. 10 should be vanishingly unlikely to result except by some
external force like unbalanced re-replication or the addition of a
tablet server, so it should suffice as a threshold.

Change-Id: I107256de604998cbf9206a8fccb3a43de86f81a8
Reviewed-on: http://gerrit.cloudera.org:8080/10787
Tested-by: Will Berkeley 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/72d0981a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/72d0981a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/72d0981a

Branch: refs/heads/master
Commit: 72d0981aa5930d3e0762dad988a991b35603b2f9
Parents: 9c89c2a
Author: Will Berkeley 
Authored: Thu Jun 21 12:21:41 2018 -0700
Committer: Will Berkeley 
Committed: Wed Jun 27 18:26:52 2018 +

--
 src/kudu/master/master.cc |  2 +-
 src/kudu/master/ts_manager.cc | 35 +++-
 src/kudu/master/ts_manager.h  |  8 ++-
 src/kudu/scripts/max_skew_estimate.py | 87 ++
 4 files changed, 129 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/72d0981a/src/kudu/master/master.cc
--
diff --git a/src/kudu/master/master.cc b/src/kudu/master/master.cc
index a6210e8..61efd76 100644
--- a/src/kudu/master/master.cc
+++ b/src/kudu/master/master.cc
@@ -118,7 +118,7 @@ GROUP_FLAG_VALIDATOR(hive_metastore_sasl_enabled, 
ValidateHiveMetastoreSaslEnabl
 Master::Master(const MasterOptions& opts)
   : KuduServer("Master", opts, "kudu.master"),
 state_(kStopped),
-ts_manager_(new TSManager()),
+ts_manager_(new TSManager(metric_entity_)),
 catalog_manager_(new CatalogManager(this)),
 path_handlers_(new MasterPathHandlers(this)),
 opts_(opts),

http://git-wip-us.apache.org/repos/asf/kudu/blob/72d0981a/src/kudu/master/ts_manager.cc
--
diff --git a/src/kudu/master/ts_manager.cc b/src/kudu/master/ts_manager.cc
index 8b1aa1c..39e4df6 100644
--- a/src/kudu/master/ts_manager.cc
+++ b/src/kudu/master/ts_manager.cc
@@ -17,17 +17,30 @@
 
 #include "kudu/master/ts_manager.h"
 
+#include 
+#include 
 #include 
 #include 
 
 #include 
 
 #include "kudu/common/wire_protocol.pb.h"
+#include "kudu/gutil/bind.h"
+#include "kudu/gutil/bind_helpers.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/master/ts_descriptor.h"
+#include "kudu/util/metrics.h"
 #include "kudu/util/pb_util.h"
 
+METRIC_DEFINE_gauge_int32(server, cluster_replica_skew,
+  "Cluster Replica Skew",
+  kudu::MetricUnit::kTablets,
+  "The difference between the number of replicas on "
+  "the tablet server hosting the most replicas and "
+  "the number of replicas on the tablet server hosting 
"
+  "the least replicas.");
+
 using std::shared_ptr;
 using std::string;
 using std::vector;
@@ -36,7 +49,11 @@ using strings::Substitute;
 namespace kudu {
 namespace master {
 
-TSManager::TSManager() {
+TSManager::TSManager(const scoped_refptr& metric_entity) {
+  METRIC_cluster_replica_skew.InstantiateFunctionGauge(
+  metric_entity,
+  

kudu git commit: KUDU-2471: Fix ColumnSchema.equals NPE with non-Decimal columns

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.7.x 8d7221570 -> a307f2a31


KUDU-2471: Fix ColumnSchema.equals NPE with non-Decimal columns

Fixes a potential NPE on non-decimal columns and adds
tests to prevent the issue in the future.

Change-Id: Ica7afcbbbd9197e6ffd34761a5298f29e7306dd6
Reviewed-on: http://gerrit.cloudera.org:8080/10826
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 
(cherry picked from commit 81828ba64f6348da2a4aec3fab3e7833f7ad01f0)
Reviewed-on: http://gerrit.cloudera.org:8080/10837
Tested-by: Grant Henke 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a307f2a3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a307f2a3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a307f2a3

Branch: refs/heads/branch-1.7.x
Commit: a307f2a31c19f96932e608d1aecfc0dd471c9a9e
Parents: 8d72215
Author: Grant Henke 
Authored: Tue Jun 26 15:00:39 2018 -0500
Committer: Grant Henke 
Committed: Wed Jun 27 18:37:28 2018 +

--
 .../main/java/org/apache/kudu/ColumnSchema.java | 28 +++--
 .../java/org/apache/kudu/TestColumnSchema.java  | 44 
 2 files changed, 50 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a307f2a3/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
--
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java 
b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
index 3072076..964c50e 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
@@ -191,29 +191,13 @@ public class ColumnSchema {
 
   @Override
   public boolean equals(Object o) {
-if (this == o) {
-  return true;
-}
-if (o == null || getClass() != o.getClass()) {
-  return false;
-}
-
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
 ColumnSchema that = (ColumnSchema) o;
-
-if (key != that.key) {
-  return false;
-}
-if (!name.equals(that.name)) {
-  return false;
-}
-if (!type.equals(that.type)) {
-  return false;
-}
-if (!typeAttributes.equals(that.typeAttributes)) {
-  return false;
-}
-
-return true;
+return Objects.equals(name, that.name) &&
+Objects.equals(type, that.type) &&
+Objects.equals(key, that.key) &&
+Objects.equals(typeAttributes, that.typeAttributes);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/kudu/blob/a307f2a3/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java 
b/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
index d2d0710..51d5cb8 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
@@ -21,6 +21,7 @@ import org.apache.kudu.util.DecimalUtil;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 
 public class TestColumnSchema {
 
@@ -36,4 +37,47 @@ public class TestColumnSchema {
 assertEquals("Column name: col2, type: int64", col2.toString());
 assertEquals("Column name: col3, type: decimal(5, 2)", col3.toString());
   }
+
+  @Test
+  public void testEquals() {
+ColumnSchema stringCol1 = new ColumnSchemaBuilder("col1", Type.STRING)
+.defaultValue("test")
+.build();
+// Same instance
+assertEquals(stringCol1, stringCol1);
+
+// Same value
+ColumnSchema stringCol2 = new ColumnSchemaBuilder("col1", Type.STRING)
+.defaultValue("test")
+.build();
+assertEquals(stringCol1, stringCol2);
+
+// Different by key
+ColumnSchema isKey = new ColumnSchemaBuilder("col1", Type.STRING)
+.key(true)
+.build();
+assertNotEquals(stringCol1, isKey);
+
+// Different by type
+ColumnSchema isInt = new ColumnSchemaBuilder("col1", Type.INT32)
+.build();
+assertNotEquals(stringCol1, isInt);
+
+// Same with type attributes
+ColumnSchema decCol1 = new ColumnSchemaBuilder("col1", Type.DECIMAL)
+.typeAttributes(DecimalUtil.typeAttributes(9, 2))
+.build();
+ColumnSchema decCol2 = new ColumnSchemaBuilder("col1", Type.DECIMAL)
+.typeAttributes(DecimalUtil.typeAttributes(9, 2))
+.build();
+assertEquals(decCol1, decCol2);
+
+// Different by type attributes
+ColumnSchema decCol3 = new ColumnSchemaBuilder("col1", Type.DECIMAL)
+

[2/3] kudu git commit: [consensus] update UnsafeChangeConfig() signature

[danburkert]

[consensus] update UnsafeChangeConfig() signature

While testing various scenarios involving kudu CLI's
'remote_replica unsafe_change_config' sub-command, it turned out that
tserver might end up trying to respond with invalid value of
TabletServerErrorPB::Code.  In that case, a tserver built in DEBUG
configuration would crash with the following stack trace:

  ../nptl/sysdeps/unix/sysv/linux/raise.c:64
assertion=0x7fb4eab7c4f8
  "::kudu::tserver::TabletServerErrorPB_Code_IsValid(value)",
file=0x7fb4eab7c531 "src/kudu/tserver/tserver.pb.h",
line=, function=) at assert.c:96
assertion=0x7fb4eab7c4f8
  "::kudu::tserver::TabletServerErrorPB_Code_IsValid(value)",
file=0x7fb4eab7c531 "src/kudu/tserver/tserver.pb.h", line=2623,
function=0x7fb4eab7c586 "void kudu::tserver::TabletServerErrorPB::set_code(
  ::kudu::tserver::TabletServerErrorPB_Code)") at assert.c:105
  kudu::tserver::TabletServerErrorPB_Code) ()
at src/kudu/tserver/tserver.pb.h:2623
  kudu::tserver::TabletServerErrorPB*,
  kudu::Status const&, kudu::tserver::TabletServerErrorPB_Code,
  kudu::rpc::RpcContext*) ()
at src/kudu/tserver/tablet_service.cc:370
  kudu::consensus::UnsafeChangeConfigRequestPB const*,
  kudu::consensus::UnsafeChangeConfigResponsePB*,
  kudu::rpc::RpcContext*) () at src/kudu/tserver/tablet_service.cc:1060

This changelist fixes the issue, 'unifying' the signatures of the
RaftConsensus::{ChangeConfig,BulkChangeConfig,UpdateChangeConfig}()
methods.

Change-Id: Id266dd72ef5e44e9b4e2ce9df10694bce8cfe2fb
Reviewed-on: http://gerrit.cloudera.org:8080/10141
Tested-by: Alexey Serbin 
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f1e652b2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f1e652b2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f1e652b2

Branch: refs/heads/master
Commit: f1e652b23c2bd76033868fe6a30484cb30098649
Parents: 009cfe2
Author: Alexey Serbin 
Authored: Fri Apr 20 18:37:25 2018 -0700
Committer: Alexey Serbin 
Committed: Fri Jun 22 18:05:06 2018 +

--
 src/kudu/consensus/raft_consensus.cc | 22 ++
 src/kudu/consensus/raft_consensus.h  |  2 +-
 src/kudu/tserver/tablet_service.cc   | 10 ++
 3 files changed, 17 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f1e652b2/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 68f1859..b4ef789 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -1886,8 +1886,9 @@ Status RaftConsensus::BulkChangeConfig(const 
BulkChangeConfigRequestPB& req,
   return Status::OK();
 }
 
-Status RaftConsensus::UnsafeChangeConfig(const UnsafeChangeConfigRequestPB& 
req,
- TabletServerErrorPB::Code* 
error_code) {
+Status RaftConsensus::UnsafeChangeConfig(
+const UnsafeChangeConfigRequestPB& req,
+boost::optional* error_code) {
   if (PREDICT_FALSE(!req.has_new_config())) {
 *error_code = TabletServerErrorPB::INVALID_CONFIG;
 return Status::InvalidArgument("Request must contain 'new_config' argument 
"
@@ -1960,6 +1961,7 @@ Status RaftConsensus::UnsafeChangeConfig(const 
UnsafeChangeConfigRequestPB& req,
   // in the committed config, it is rare and a replica without itself
   // in the latest config is definitely not caught up with the latest leader's 
log.
   if (!IsRaftConfigVoter(peer_uuid(), new_config)) {
+*error_code = TabletServerErrorPB::INVALID_CONFIG;
 return Status::InvalidArgument(Substitute("Local replica uuid $0 is not "
   "a VOTER in the new config, "
   "rejecting the unsafe config "
@@ -1985,13 +1987,12 @@ Status RaftConsensus::UnsafeChangeConfig(const 
UnsafeChangeConfigRequestPB& req,
   // Prepare the consensus request as if the request is being generated
   // from a different leader.
   ConsensusRequestPB consensus_req;
-  ConsensusResponsePB consensus_resp;
   consensus_req.set_caller_uuid(req.caller_id());
   // Bumping up the term for the consensus request being generated.
   // This makes this request appear to come from a new leader that
   // the local replica doesn't know about yet. If the local replica
   // happens to be the leader, this will cause it to step down.
-  int64_t new_term = current_term + 1;
+  const int64_t new_term = current_term + 1;
   consensus_req.set_caller_term(new_term);
   consensus_req.mutable_preceding_id()->CopyFrom(preceding_opid);
   consensus_req.set_committed_index(last_committed_index);
@@ -2018,14 +2019,11 @@ 

[1/3] kudu git commit: KUDU-2260: Log block manager should handle null bytes in metadata on crash

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master c7918e87b -> f6e8fe6c6


KUDU-2260: Log block manager should handle null bytes in metadata on crash

On ext4 with data=ordered (the default), it's possible for a write to
persist an increase to the filesize without persisting the actual data.
In this case, the file will contain null bytes at the end. In the LBM,
we considered this case to be corruption if there were enough null bytes
(>= 8) for a PB container record length and length checksum. However,
it's safe to call this an incomplete record and truncate the file at the
end of the last complete record.

This patch changes the P container reader code to return
Status::Incomplete if it encounters this situation. This will cause the
LBM to repair the container metadata appropriately.

Two regression tests, at the PB container file and LBM layers, are
included.

Change-Id: I0af5c9dbbe28afe7a179595ad20392b99cde2a1b
Reviewed-on: http://gerrit.cloudera.org:8080/10777
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/009cfe23
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/009cfe23
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/009cfe23

Branch: refs/heads/master
Commit: 009cfe23ab37e8c8ac5949f365ab424f34a4f520
Parents: c7918e8
Author: Will Berkeley 
Authored: Tue Jun 19 17:13:02 2018 -0700
Committer: Will Berkeley 
Committed: Fri Jun 22 17:58:01 2018 +

--
 src/kudu/consensus/log_util.cc| 27 
 src/kudu/fs/log_block_manager-test.cc | 50 +-
 src/kudu/util/pb_util-test.cc | 34 
 src/kudu/util/pb_util.cc  | 43 +++--
 src/kudu/util/slice.cc| 20 
 src/kudu/util/slice.h | 11 +++
 6 files changed, 134 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/009cfe23/src/kudu/consensus/log_util.cc
--
diff --git a/src/kudu/consensus/log_util.cc b/src/kudu/consensus/log_util.cc
index 07fba47..0f0fd23 100644
--- a/src/kudu/consensus/log_util.cc
+++ b/src/kudu/consensus/log_util.cc
@@ -454,33 +454,6 @@ Status 
ReadableLogSegment::ReadHeaderMagicAndHeaderLength(uint32_t *len) {
   return Status::OK();
 }
 
-namespace {
-
-// We don't run TSAN on this function because it makes it really slow and 
causes some
-// test timeouts. This is only used on local buffers anyway, so we don't lose 
much
-// by not checking it.
-ATTRIBUTE_NO_SANITIZE_THREAD
-bool IsAllZeros(const Slice& s) {
-  // Walk a pointer through the slice instead of using s[i]
-  // since this is way faster in debug mode builds. We also do some
-  // manual unrolling for the same purpose.
-  const uint8_t* p = [0];
-  int rem = s.size();
-
-  while (rem >= 8) {
-if (UNALIGNED_LOAD64(p) != 0) return false;
-rem -= 8;
-p += 8;
-  }
-
-  while (rem > 0) {
-if (*p++ != '\0') return false;
-rem--;
-  }
-  return true;
-}
-} // anonymous namespace
-
 Status ReadableLogSegment::ParseHeaderMagicAndHeaderLength(const Slice ,
uint32_t 
*parsed_len) {
   RETURN_NOT_OK_PREPEND(data.check_size(kLogSegmentHeaderMagicAndHeaderLength),

http://git-wip-us.apache.org/repos/asf/kudu/blob/009cfe23/src/kudu/fs/log_block_manager-test.cc
--
diff --git a/src/kudu/fs/log_block_manager-test.cc 
b/src/kudu/fs/log_block_manager-test.cc
index d93d8aa..3dc79e8 100644
--- a/src/kudu/fs/log_block_manager-test.cc
+++ b/src/kudu/fs/log_block_manager-test.cc
@@ -622,33 +622,39 @@ TEST_F(LogBlockManagerTest, TestMetadataTruncation) {
   uint64_t good_meta_size;
   ASSERT_OK(env_->GetFileSize(metadata_path, _meta_size));
 
-  // First, add an extra byte to the end of the metadata file. This makes the
-  // trailing "record" of the metadata file corrupt, but doesn't cause data
+  // First, add extra null bytes to the end of the metadata file. This makes
+  // the trailing "record" of the metadata file corrupt, but doesn't cause data
   // loss. The result is that the container will automatically truncate the
   // metadata file back to its correct size.
-  {
-RWFileOptions opts;
-opts.mode = Env::OPEN_EXISTING;
-unique_ptr file;
-ASSERT_OK(env_->NewRWFile(opts, metadata_path, ));
-ASSERT_OK(file->Truncate(good_meta_size + 1));
-  }
-
+  // We'll do this with 1, 8, and 128 extra bytes-- the first case is too few
+  // bytes to be a valid record, while the second is too few but is enough for
+  // a data length and its checksum, and the third is too long for a record.
+  // The 8- and 128-byte cases are 

[3/3] kudu git commit: Synchronizer::WaitFor thread-safety

[danburkert]

Synchronizer::WaitFor thread-safety

WaitFor's implementation previously was not thread safe in the case that
the waiter deallocated the Synchronizer after receiving a timeout.

Change-Id: I9565a50839ffd23b5bac6986a6fdee41ac21aa3a
Reviewed-on: http://gerrit.cloudera.org:8080/10783
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f6e8fe6c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f6e8fe6c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f6e8fe6c

Branch: refs/heads/master
Commit: f6e8fe6c6bb6dd83e6485bc1b56364b11b250e94
Parents: f1e652b
Author: Dan Burkert 
Authored: Thu Jun 21 11:49:39 2018 -0700
Committer: Dan Burkert 
Committed: Fri Jun 22 22:15:55 2018 +

--
 src/kudu/util/CMakeLists.txt |   1 +
 src/kudu/util/async_util-test.cc | 129 ++
 src/kudu/util/async_util.h   |  71 +++
 3 files changed, 170 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f6e8fe6c/src/kudu/util/CMakeLists.txt
--
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index 044660c..743e1af 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -331,6 +331,7 @@ target_link_libraries(protoc-gen-insertions gutil protobuf 
protoc ${KUDU_BASE_LI
 ###
 
 set(KUDU_TEST_LINK_LIBS kudu_util gutil ${KUDU_MIN_TEST_LIBS})
+ADD_KUDU_TEST(async_util-test)
 ADD_KUDU_TEST(atomic-test)
 ADD_KUDU_TEST(bit-util-test)
 ADD_KUDU_TEST(bitmap-test)

http://git-wip-us.apache.org/repos/asf/kudu/blob/f6e8fe6c/src/kudu/util/async_util-test.cc
--
diff --git a/src/kudu/util/async_util-test.cc b/src/kudu/util/async_util-test.cc
new file mode 100644
index 000..5cb7a63
--- /dev/null
+++ b/src/kudu/util/async_util-test.cc
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/util/async_util.h"
+
+#include 
+
+#include 
+#include 
+#include 
+
+#include 
+
+#include "kudu/gutil/basictypes.h"
+#include "kudu/gutil/callback.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+using std::thread;
+using std::vector;
+
+namespace kudu {
+
+class AsyncUtilTest : public KuduTest {
+ public:
+  AsyncUtilTest() {
+// Set up an alarm to fail the test in case of deadlock.
+alarm(30);
+  }
+  ~AsyncUtilTest() {
+// Disable the alarm on test exit.
+alarm(0);
+  }
+};
+
+// Test completing the synchronizer through each of the APIs it exposes.
+TEST_F(AsyncUtilTest, TestSynchronizerCompletion) {
+  Synchronizer sync;
+
+  {
+auto waiter = thread([sync] {
+ignore_result(sync.Wait());
+});
+SleepFor(MonoDelta::FromMilliseconds(5));
+sync.StatusCB(Status::OK());
+waiter.join();
+  }
+  sync.Reset();
+  {
+auto cb = sync.AsStatusCallback();
+auto waiter = thread([sync] {
+ignore_result(sync.Wait());
+});
+SleepFor(MonoDelta::FromMilliseconds(5));
+cb.Run(Status::OK());
+waiter.join();
+  }
+  sync.Reset();
+  {
+auto cb = sync.AsStdStatusCallback();
+auto waiter = thread([sync] {
+ignore_result(sync.Wait());
+});
+SleepFor(MonoDelta::FromMilliseconds(5));
+cb(Status::OK());
+waiter.join();
+  }
+}
+
+TEST_F(AsyncUtilTest, TestSynchronizerMultiWait) {
+  Synchronizer sync;
+  vector waiters;
+  for (int i = 0; i < 5; i++) {
+waiters.emplace_back([sync] {
+ignore_result(sync.Wait());
+});
+  }
+  SleepFor(MonoDelta::FromMilliseconds(5));
+  sync.StatusCB(Status::OK());
+
+  for (auto& waiter : waiters) {
+waiter.join();
+  }
+}
+
+TEST_F(AsyncUtilTest, TestSynchronizerTimedWait) {
+  thread waiter;
+  {
+Synchronizer sync;
+auto cb = sync.AsStatusCallback();
+waiter = thread([cb] {
+

kudu git commit: KUDU-2447 Fix TS crashed with "NONE predicate can not be pushed into key"

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 61fb6dc30 -> c7918e87b


KUDU-2447 Fix TS crashed with "NONE predicate can not be pushed into key"

tserver crashes on Scan when [lowerPrimaryKey, upperPrimary)
 and predicates of primary column do not overlap

Change-Id: I02810d651e14768f0a85044b0b7f01fa7246c718
Reviewed-on: http://gerrit.cloudera.org:8080/10514
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c7918e87
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c7918e87
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c7918e87

Branch: refs/heads/master
Commit: c7918e87bf1aeb25d5b08f7b340d1ae15806568d
Parents: 61fb6dc
Author: oclarms 
Authored: Fri May 25 19:55:36 2018 +0800
Committer: Dan Burkert 
Committed: Thu Jun 21 20:15:22 2018 +

--
 .../kudu/client/TestScannerMultiTablet.java |  7 +-
 src/kudu/common/scan_spec-test.cc   | 81 ++--
 src/kudu/common/scan_spec.cc|  5 +-
 3 files changed, 67 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c7918e87/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
index 7a7fb7e..d4a0c5f 100644
--- 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
+++ 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestScannerMultiTablet.java
@@ -141,8 +141,13 @@ public class TestScannerMultiTablet extends BaseKuduTest {
   // Test mixing start/end row keys with predicates.
   @Test(timeout = 10)
   public void testKeysAndPredicates() throws Exception {
+// Value that doesn't exist, predicates has primary column
+ColumnRangePredicate predicate = new 
ColumnRangePredicate(schema.getColumnByIndex(1));
+predicate.setUpperBound("1");
+assertEquals(0, countRowsInScan(getScanner("1", "2", "1", "3", 
predicate)));
+
 // First row from the 2nd tablet.
-ColumnRangePredicate predicate = new 
ColumnRangePredicate(schema.getColumnByIndex(2));
+predicate = new ColumnRangePredicate(schema.getColumnByIndex(2));
 predicate.setLowerBound("1");
 predicate.setUpperBound("1");
 assertEquals(1, countRowsInScan(getScanner("1", "", "2", "", predicate)));

http://git-wip-us.apache.org/repos/asf/kudu/blob/c7918e87/src/kudu/common/scan_spec-test.cc
--
diff --git a/src/kudu/common/scan_spec-test.cc 
b/src/kudu/common/scan_spec-test.cc
index 8835dc7..79969bd 100644
--- a/src/kudu/common/scan_spec-test.cc
+++ b/src/kudu/common/scan_spec-test.cc
@@ -642,34 +642,67 @@ TEST_F(CompositeIntKeysTest, 
TestLiftPrimaryKeyBounds_BothBounds) {
 // Test that implicit constraints specified in the primary key upper/lower
 // bounds are merged into the set of predicates.
 TEST_F(CompositeIntKeysTest, TestLiftPrimaryKeyBounds_WithPredicates) {
-  // b >= 15
-  // c >= 3
-  // c <= 100
-  // key >= (10, min, min)
-  //  < (10,  90, min)
-  ScanSpec spec;
-  AddPredicate(, "b", GE, 15);
-  AddPredicate(, "c", GE, 3);
-  AddPredicate(, "c", LE, 100);
+  {
+// b >= 15
+// c >= 3
+// c <= 100
+// key >= (10, min, min)
+//  < (10,  90, min)
+ScanSpec spec;
+AddPredicate(, "b", GE, 15);
+AddPredicate(, "c", GE, 3);
+AddPredicate(, "c", LE, 100);
 
-  KuduPartialRow lower_bound(_);
-  CHECK_OK(lower_bound.SetInt8("a", 10));
-  CHECK_OK(lower_bound.SetInt8("b", INT8_MIN));
-  CHECK_OK(lower_bound.SetInt8("c", INT8_MIN));
+KuduPartialRow lower_bound(_);
+CHECK_OK(lower_bound.SetInt8("a", 10));
+CHECK_OK(lower_bound.SetInt8("b", INT8_MIN));
+CHECK_OK(lower_bound.SetInt8("c", INT8_MIN));
 
-  KuduPartialRow upper_bound(_);
-  CHECK_OK(upper_bound.SetInt8("a", 10));
-  CHECK_OK(upper_bound.SetInt8("b", 90));
-  CHECK_OK(upper_bound.SetInt8("c", INT8_MIN));
+KuduPartialRow upper_bound(_);
+CHECK_OK(upper_bound.SetInt8("a", 10));
+CHECK_OK(upper_bound.SetInt8("b", 90));
+CHECK_OK(upper_bound.SetInt8("c", INT8_MIN));
 
-  SetLowerBound(, lower_bound);
-  SetExclusiveUpperBound(, upper_bound);
+SetLowerBound(, lower_bound);
+SetExclusiveUpperBound(, upper_bound);
 
-  spec.OptimizeScan(schema_, _, _, false);
-  ASSERT_EQ(3, spec.predicates().size());
-  ASSERT_EQ("a = 10", FindOrDie(spec.predicates(), "a").ToString());
-  ASSERT_EQ("b >= 15 AND b < 90", FindOrDie(spec.predicates(), 
"b").ToString());
-  ASSERT_EQ("c >= 3 AND c < 101", FindOrDie(spec.predicates(), 

[1/2] kudu git commit: KUDU-2191 (12/n): Hive Metastore notification log event listener

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master b5f3d1a10 -> 5f1ca32f3


http://git-wip-us.apache.org/repos/asf/kudu/blob/5f1ca32f/src/kudu/master/hms_notification_log_listener.h
--
diff --git a/src/kudu/master/hms_notification_log_listener.h 
b/src/kudu/master/hms_notification_log_listener.h
new file mode 100644
index 000..c4e152d
--- /dev/null
+++ b/src/kudu/master/hms_notification_log_listener.h
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include 
+#include 
+
+#include "kudu/gutil/port.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/util/condition_variable.h"
+#include "kudu/util/mutex.h"
+#include "kudu/util/status.h"
+#include "kudu/util/status_callback.h"
+
+namespace hive {
+class NotificationEvent;
+}
+
+namespace kudu {
+
+class MonoTime;
+class Thread;
+
+namespace master {
+
+class CatalogManager;
+
+// A CatalogManager background task which listens for events occurring in the
+// Hive Metastore, and synchronizes the Kudu catalog accordingly.
+//
+// As a background task, the lifetime of an instance of this class must be less
+// than the catalog manager it belongs to.
+//
+// The notification log listener task continuously wakes up according to its
+// configured poll period, however it performs no work when the master is a
+// follower.
+//
+// When a change to the Kudu catalog is performed in response to a notification
+// log event, the corresponding event ID is recorded in the sys catalog as the
+// latest handled event. This ensures that masters do not double-apply
+// notification events as leadership changes.
+//
+// The notification log listener listens for two types of events on Kudu 
tables:
+//
+// - ALTER TABLE RENAME
+//Table rename is a special case of ALTER TABLE. The notification log
+//listener listens for rename event notifications for Kudu tables, and
+//renames the corresponding Kudu table. See below for why renames can be
+//applied back to Kudu, but not other types of alterations.
+//
+// - DROP TABLE
+//The notification log listener listens for drop table events for Kudu
+//tables, and drops the corresponding Kudu table. This allows the catalogs
+//to stay synchronized when DROP TABLE and DROP DATABASE CASCADE Hive
+//commands are executed.
+//
+// The notification log listener can support renaming and dropping tables in a
+// safe manner because the Kudu table ID is stored in the HMS table entry. 
Using
+// the Kudu table ID, the exact table which the event applies to can always be
+// identified. For other changes made in ALTER TABLE statements, such as ALTER
+// TABLE DROP COLUMN, there is no way to identify with certainty which column
+// has been dropped, since we do not store column IDs in the HMS table entries.
+class HmsNotificationLogListenerTask {
+ public:
+
+  explicit HmsNotificationLogListenerTask(CatalogManager* catalog_manager);
+  ~HmsNotificationLogListenerTask();
+
+  // Initializes the HMS notification log listener. When invoking this method,
+  // the catalog manager must be in the process of initializing.
+  Status Init() WARN_UNUSED_RESULT;
+
+  // Shuts down the HMS notification log listener. This must be called before
+  // shutting down the catalog manager.
+  void Shutdown();
+
+  // Waits for the notification log listener to process the latest notification
+  // log event.
+  //
+  // Note: an error will be returned if the listener is unable to retrieve the
+  // latest notifications from the HMS. If individual notifications are unable
+  // to be processed, no error will be returned.
+  Status WaitForCatchUp(const MonoTime& deadline) WARN_UNUSED_RESULT;
+
+ private:
+
+  // Runs the main loop of the listening thread.
+  void RunLoop();
+
+  // Polls the Hive Metastore for notification events, and handle them.
+  Status Poll();
+
+  // Handles an ALTER TABLE event. Must only be called on the listening thread.
+  //
+  // The event is parsed, and if it is a rename table event for a Kudu table,
+  // the table is renamed in the local catalog.  All other events are ignored.
+  Status HandleAlterTableEvent(const 

kudu git commit: Bump Hive version

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master c1c15ad5f -> 930c7f40f


Bump Hive version

HIVE-16886/HIVE-18526 is causing the notification log listener tests to
be flaky, this version bump includes the fix committed upstream.

Change-Id: I5f2872da67e718d0801f08a71482a6bf2ecb95d2
Reviewed-on: http://gerrit.cloudera.org:8080/10701
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/930c7f40
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/930c7f40
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/930c7f40

Branch: refs/heads/master
Commit: 930c7f40fc18be16b2a905ea7f5d630c8e124fe0
Parents: c1c15ad
Author: Dan Burkert 
Authored: Thu Jun 7 14:23:28 2018 -0700
Committer: Dan Burkert 
Committed: Wed Jun 13 02:26:09 2018 +

--
 thirdparty/vars.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/930c7f40/thirdparty/vars.sh
--
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index ebc01ef..0483906 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -208,11 +208,11 @@ BISON_VERSION=3.0.4
 BISON_NAME=bison-$BISON_VERSION
 BISON_SOURCE=$TP_SOURCE_DIR/$BISON_NAME
 
-# TODO(dan): bump to a release version once HIVE-17747 is published. The SHA
-# below is the current head of branch-2.
+# TODO(dan): bump to a release version once HIVE-17747 and 
HIVE-16886/HIVE-18526
+# are published. The SHA below is the current head of branch-2.
 # Note: The Hive release binary tarball is stripped of unnecessary jars before
 # being uploaded. See thirdparty/package-hive.sh for details.
-HIVE_VERSION=6189dbdea543939e977f97170f9a37389f46eb29
+HIVE_VERSION=c0a57a33b44d73fbe0f95256f0a3822bd3720aa8
 HIVE_NAME=apache-hive-$HIVE_VERSION-bin
 HIVE_SOURCE=$TP_SOURCE_DIR/$HIVE_NAME
 

kudu git commit: KUDU-2472: address master-stress-test flakiness

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 37f3a95d8 -> c1c15ad5f


KUDU-2472: address master-stress-test flakiness

Restores a hack that prevented master-stress-test flakiness, and adds a
TODO to remove the hack once KUDU-1358 is addressed. The hack was broken
in 1a707b991 when an error message was tweaked.

Change-Id: I639eedaf2efc50d1be0dc719f2fcb7cc8fdd1fb9
Reviewed-on: http://gerrit.cloudera.org:8080/10700
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c1c15ad5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c1c15ad5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c1c15ad5

Branch: refs/heads/master
Commit: c1c15ad5fd6dd1d96dae4695880680f91130d7a0
Parents: 37f3a95
Author: Dan Burkert 
Authored: Tue Jun 12 10:55:42 2018 -0700
Committer: Dan Burkert 
Committed: Tue Jun 12 18:53:01 2018 +

--
 src/kudu/integration-tests/master-stress-test.cc | 10 +++---
 src/kudu/master/catalog_manager.cc   |  1 +
 2 files changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c1c15ad5/src/kudu/integration-tests/master-stress-test.cc
--
diff --git a/src/kudu/integration-tests/master-stress-test.cc 
b/src/kudu/integration-tests/master-stress-test.cc
index 30bed01..9dfd600 100644
--- a/src/kudu/integration-tests/master-stress-test.cc
+++ b/src/kudu/integration-tests/master-stress-test.cc
@@ -211,9 +211,13 @@ class MasterStressTest : public KuduTest {
 continue;
   }
   if (s.IsInvalidArgument() &&
-  MatchPattern(s.ToString(), "*Not enough live tablet servers*")) {
-// The test placed enough load on the cluster that some tservers
-// haven't heartbeat in a little while.
+  MatchPattern(s.ToString(), "*not enough live tablet servers*")) {
+// The test placed enough load on the cluster that some tservers 
haven't
+// heartbeat in a while, or the leader master has not been alive for
+// long enough to receive heartbeats from all tservers.
+//
+// TODO(KUDU-1358): remove this special case once table creation
+// following leader restart is robust.
 continue;
   }
   CHECK_OK(s);

http://git-wip-us.apache.org/repos/asf/kudu/blob/c1c15ad5/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 6438901..55cf535 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -1438,6 +1438,7 @@ Status CatalogManager::CreateTable(const 
CreateTableRequestPB* orig_req,
   // Verify that the number of replicas isn't larger than the number of live 
tablet
   // servers.
   if (FLAGS_catalog_manager_check_ts_count_for_create_table && num_replicas > 
num_live_tservers) {
+// Note: this error message is matched against in master-stress-test.
 return SetupError(Status::InvalidArgument(Substitute(
 "not enough live tablet servers to create a table with the 
requested replication "
 "factor $0; $1 tablet servers are alive", req.num_replicas(), 
num_live_tservers)),

kudu git commit: Rename Hive MetaStore to Metastore

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master ca3e38759 -> 5d113ec3c


Rename Hive MetaStore to Metastore

Hive isn't consistent about the capitilization so it's not clear which
is more correct, but we should at least be consistent ourselves.

Change-Id: I47c0a6534de9d612787c23e2886b4150e3b69527
Reviewed-on: http://gerrit.cloudera.org:8080/10379
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5d113ec3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5d113ec3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5d113ec3

Branch: refs/heads/master
Commit: 5d113ec3c3a7296cc7a704bad415c74aab9f49fa
Parents: ca3e387
Author: Dan Burkert 
Authored: Fri May 11 10:41:13 2018 -0700
Committer: Dan Burkert 
Committed: Mon May 14 19:44:24 2018 +

--
 src/kudu/hms/hms_client.cc | 30 +++---
 src/kudu/hms/hms_client.h  |  2 +-
 src/kudu/hms/mini_hms.cc   |  6 +++---
 3 files changed, 19 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/5d113ec3/src/kudu/hms/hms_client.cc
--
diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc
index ca6fdb8..3f1f948 100644
--- a/src/kudu/hms/hms_client.cc
+++ b/src/kudu/hms/hms_client.cc
@@ -53,7 +53,7 @@
 // Default to 100 MiB to match Thrift TSaslTransport.receiveSaslMessage and the
 // HMS metastore.server.max.message.size config.
 DEFINE_int32(hms_client_max_buf_size, 100 * 1024 * 1024,
- "Maximum size of Hive MetaStore objects that can be received by 
the "
+ "Maximum size of Hive Metastore objects that can be received by 
the "
  "HMS client in bytes.");
 TAG_FLAG(hms_client_max_buf_size, experimental);
 // Note: despite being marked as a runtime flag, the new buf size value will
@@ -176,13 +176,13 @@ HmsClient::~HmsClient() {
 Status HmsClient::Start() {
   SCOPED_LOG_SLOW_EXECUTION(WARNING, kSlowExecutionWarningThresholdMs, 
"starting HMS client");
   HMS_RET_NOT_OK(client_.getOutputProtocol()->getTransport()->open(),
- "failed to open Hive MetaStore connection");
+ "failed to open Hive Metastore connection");
 
   // Immediately after connecting to the HMS, check that it is configured with
   // the required event listeners.
   string event_listener_config;
   HMS_RET_NOT_OK(client_.get_config_value(event_listener_config, 
kTransactionalEventListeners, ""),
- Substitute("failed to get Hive MetaStore $0 configuration",
+ Substitute("failed to get Hive Metastore $0 configuration",
 kTransactionalEventListeners));
 
   // Parse the set of listeners from the configuration string.
@@ -214,7 +214,7 @@ Status HmsClient::Start() {
   
HMS_RET_NOT_OK(client_.get_config_value(disallow_incompatible_column_type_changes,
   kDisallowIncompatibleColTypeChanges,
   "false"),
- Substitute("failed to get Hive MetaStore $0 configuration",
+ Substitute("failed to get Hive Metastore $0 configuration",
 kDisallowIncompatibleColTypeChanges));
 
   if (boost::iequals(disallow_incompatible_column_type_changes, "true")) {
@@ -229,7 +229,7 @@ Status HmsClient::Start() {
 Status HmsClient::Stop() {
   SCOPED_LOG_SLOW_EXECUTION(WARNING, kSlowExecutionWarningThresholdMs, 
"stopping HMS client");
   HMS_RET_NOT_OK(client_.getInputProtocol()->getTransport()->close(),
- "failed to close Hive MetaStore connection");
+ "failed to close Hive Metastore connection");
   return Status::OK();
 }
 
@@ -239,14 +239,14 @@ bool HmsClient::IsConnected() {
 
 Status HmsClient::CreateDatabase(const hive::Database& database) {
   SCOPED_LOG_SLOW_EXECUTION(WARNING, kSlowExecutionWarningThresholdMs, "create 
HMS database");
-  HMS_RET_NOT_OK(client_.create_database(database), "failed to create Hive 
MetaStore database");
+  HMS_RET_NOT_OK(client_.create_database(database), "failed to create Hive 
Metastore database");
   return Status::OK();
 }
 
 Status HmsClient::DropDatabase(const string& database_name, Cascade cascade) {
   SCOPED_LOG_SLOW_EXECUTION(WARNING, kSlowExecutionWarningThresholdMs, "drop 
HMS database");
   HMS_RET_NOT_OK(client_.drop_database(database_name, true, cascade == 
Cascade::kTrue),
- "failed to drop Hive MetaStore database");
+ "failed to drop Hive Metastore database");
   return Status::OK();
 }
 
@@ -254,7 +254,7 @@ Status HmsClient::GetAllDatabases(vector* 
databases) {
   DCHECK(databases);
   

kudu git commit: catalog-manager: refactor AlterTable and DeleteTable methods

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master a65e58ec0 -> ca3e38759


catalog-manager: refactor AlterTable and DeleteTable methods

This commit splits CatalogManager::AlterTable and
CatalogManager::DeleteTable in two. One method handles RPC specifics,
and the second handles applying the alter/delete operation to the Kudu
catalog. The RPC-handling method thus calls into the method which
modifies the catalog.  A follow-up commit in the HMS integration series
will add another front-end method specific to HMS notification log
listener events.

Change-Id: Ia384768ee7246411052ccadc66c33e83b541c195
Reviewed-on: http://gerrit.cloudera.org:8080/10378
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ca3e3875
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ca3e3875
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ca3e3875

Branch: refs/heads/master
Commit: ca3e387591d1947c24f9af814bb6429447902db5
Parents: a65e58e
Author: Dan Burkert 
Authored: Thu May 10 13:01:33 2018 -0700
Committer: Dan Burkert 
Committed: Mon May 14 19:20:05 2018 +

--
 src/kudu/integration-tests/alter_table-test.cc |  2 +-
 src/kudu/master/catalog_manager.cc | 75 -
 src/kudu/master/catalog_manager.h  | 25 ---
 src/kudu/master/master_service.cc  |  4 +-
 4 files changed, 62 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ca3e3875/src/kudu/integration-tests/alter_table-test.cc
--
diff --git a/src/kudu/integration-tests/alter_table-test.cc 
b/src/kudu/integration-tests/alter_table-test.cc
index f771209..640f1cb 100644
--- a/src/kudu/integration-tests/alter_table-test.cc
+++ b/src/kudu/integration-tests/alter_table-test.cc
@@ -344,7 +344,7 @@ TEST_F(AlterTableTest, 
TestAddNotNullableColumnWithoutDefaults) {
 cluster_->mini_master()->master()->catalog_manager();
 master::CatalogManager::ScopedLeaderSharedLock l(catalog);
 ASSERT_OK(l.first_failed_status());
-Status s = catalog->AlterTable(, , nullptr);
+Status s = catalog->AlterTableRpc(req, , nullptr);
 ASSERT_TRUE(s.IsInvalidArgument());
 ASSERT_STR_CONTAINS(s.ToString(), "column `c2`: NOT NULL columns must have 
a default");
   }

http://git-wip-us.apache.org/repos/asf/kudu/blob/ca3e3875/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 174c7fc..6438901 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -1700,19 +1700,22 @@ Status CatalogManager::FindAndLockTable(const ReqClass& 
request,
   return Status::OK();
 }
 
-Status CatalogManager::DeleteTable(const DeleteTableRequestPB* req,
-   DeleteTableResponsePB* resp,
-   rpc::RpcContext* rpc) {
+Status CatalogManager::DeleteTableRpc(const DeleteTableRequestPB& req,
+  DeleteTableResponsePB* resp,
+  rpc::RpcContext* rpc) {
+  LOG(INFO) << Substitute("Servicing DeleteTable request from $0:\n$1",
+  RequestorString(rpc), SecureShortDebugString(req));
+  return DeleteTable(req, resp);
+}
+
+Status CatalogManager::DeleteTable(const DeleteTableRequestPB& req, 
DeleteTableResponsePB* resp) {
   leader_lock_.AssertAcquiredForReading();
   RETURN_NOT_OK(CheckOnline());
 
-  LOG(INFO) << Substitute("Servicing DeleteTable request from $0:\n$1",
-  RequestorString(rpc), SecureShortDebugString(*req));
-
   // 1. Look up the table, lock it, and mark it as removed.
   scoped_refptr table;
   TableMetadataLock l;
-  RETURN_NOT_OK(FindAndLockTable(*req, resp, LockMode::WRITE, , ));
+  RETURN_NOT_OK(FindAndLockTable(req, resp, LockMode::WRITE, , ));
   if (l.data().is_deleted()) {
 return SetupError(Status::NotFound("the table was deleted", 
l.data().pb.state_msg()),
 resp, MasterErrorPB::TABLE_NOT_FOUND);
@@ -1788,7 +1791,9 @@ Status CatalogManager::DeleteTable(const 
DeleteTableRequestPB* req,
   TRACE("Removing table from by-name map");
   std::lock_guard l_map(lock_);
   if (table_names_map_.erase(l.data().name()) != 1) {
-PANIC_RPC(rpc, "Could not remove table from map, name=" + 
l.data().name());
+LOG(FATAL) << "Could not remove table " << table->ToString()
+   << " from map in response to DeleteTable request: "
+   << SecureShortDebugString(req);
   }
 }
 
@@ -2064,19 

kudu git commit: catalog-manager: return error from FindAndLockTable if table not found

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 1a707b991 -> a65e58ec0


catalog-manager: return error from FindAndLockTable if table not found

This cleans up some repeated boilerplate in the CatalogManager.

Change-Id: I4d8176d6886474a84b4c75c80f50763eabe70a6d
Reviewed-on: http://gerrit.cloudera.org:8080/10377
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a65e58ec
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a65e58ec
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a65e58ec

Branch: refs/heads/master
Commit: a65e58ec0612e6c20757fa51fe4e37056d5a4815
Parents: 1a707b9
Author: Dan Burkert 
Authored: Thu May 10 12:40:27 2018 -0700
Committer: Dan Burkert 
Committed: Mon May 14 19:17:04 2018 +

--
 src/kudu/master/catalog_manager.cc | 83 -
 src/kudu/master/catalog_manager.h  |  7 ++-
 2 files changed, 36 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a65e58ec/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 1de0be1..174c7fc 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -620,12 +620,15 @@ void CheckIfNoLongerLeaderAndSetupError(const Status& s, 
RespClass* resp) {
 template
 Status CheckIfTableDeletedOrNotRunning(TableMetadataLock* lock, RespClass* 
resp) {
   if (lock->data().is_deleted()) {
-return SetupError(Status::NotFound("the table was deleted", 
lock->data().pb.state_msg()),
-  resp, MasterErrorPB::TABLE_NOT_FOUND);
+return SetupError(Status::NotFound(
+  Substitute("table $0 was deleted", lock->data().name()),
+  lock->data().pb.state_msg()),
+resp, MasterErrorPB::TABLE_NOT_FOUND);
   }
   if (!lock->data().is_running()) {
-return SetupError(Status::ServiceUnavailable("the table is not running"),
-  resp, MasterErrorPB::TABLE_NOT_FOUND);
+return SetupError(Status::ServiceUnavailable(
+  Substitute("table $0 is not running", lock->data().name())),
+resp, MasterErrorPB::TABLE_NOT_FOUND);
   }
   return Status::OK();
 }
@@ -1601,15 +1604,9 @@ Status CatalogManager::IsCreateTableDone(const 
IsCreateTableDoneRequestPB* req,
   RETURN_NOT_OK(CheckOnline());
 
   // 1. Lookup the table and verify if it exists
-  TRACE("Looking up and locking table");
   scoped_refptr table;
   TableMetadataLock l;
-  RETURN_NOT_OK(FindAndLockTable(req->table(), LockMode::READ, , ));
-  if (table == nullptr) {
-return SetupError(
-Status::NotFound("the table does not exist", 
SecureShortDebugString(req->table())),
-resp, MasterErrorPB::TABLE_NOT_FOUND);
-  }
+  RETURN_NOT_OK(FindAndLockTable(*req, resp, LockMode::READ, , ));
   RETURN_NOT_OK(CheckIfTableDeletedOrNotRunning(, resp));
 
   // 2. Verify if the create is in-progress
@@ -1649,10 +1646,21 @@ scoped_refptr 
CatalogManager::CreateTabletInfo(const scoped_refptr
+Status CatalogManager::FindAndLockTable(const ReqClass& request,
+RespClass* response,
 LockMode lock_mode,
 scoped_refptr* table_info,
 TableMetadataLock* table_lock) {
+  TRACE("Looking up and locking table");
+  const TableIdentifierPB& table_identifier = request.table();
+
+  auto tnf_error = [&] {
+return SetupError(Status::NotFound(
+  "the table does not exist", 
SecureShortDebugString(table_identifier)),
+response, MasterErrorPB::TABLE_NOT_FOUND);
+  };
+
   scoped_refptr table;
   {
 shared_lock l(lock_);
@@ -1663,18 +1671,19 @@ Status CatalogManager::FindAndLockTable(const 
TableIdentifierPB& table_identifie
   // both match the same table.
   if (table_identifier.has_table_name() &&
   table.get() != FindPtrOrNull(table_names_map_, 
table_identifier.table_name()).get()) {
-return Status::OK();
+return tnf_error();
   }
 } else if (table_identifier.has_table_name()) {
   table = FindPtrOrNull(table_names_map_, table_identifier.table_name());
 } else {
-  return Status::InvalidArgument("Missing Table ID or Table Name");
+  return SetupError(Status::InvalidArgument("missing table ID or table 
name"),
+response, MasterErrorPB::UNKNOWN_ERROR);
 }
   }
 
   // If the table doesn't exist, don't attempt to lock it.
   if (!table) {
-return Status::OK();
+return tnf_error();
   }
 
   // Acquire the table lock.
@@ -1683,7 

kudu git commit: catalog-manager: refactor error handling

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 05d594867 -> 1a707b991


catalog-manager: refactor error handling

This commit refactors how the SetupError helper works slightly in order
to reduce the prevelance of temporary Status locals. I've found the
Status locals to be error prone in the past due to shadowing.

Change-Id: I695a62a75a3b1979741e2d7105ed131eace64b6d
Reviewed-on: http://gerrit.cloudera.org:8080/10376
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1a707b99
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1a707b99
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1a707b99

Branch: refs/heads/master
Commit: 1a707b991785a5a7d4d316d721d813f49d0bb7e5
Parents: 05d5948
Author: Dan Burkert 
Authored: Thu May 10 11:11:10 2018 -0700
Committer: Dan Burkert 
Committed: Mon May 14 18:35:12 2018 +

--
 src/kudu/client/client-test.cc |  14 +-
 src/kudu/master/catalog_manager.cc | 244 ++--
 2 files changed, 117 insertions(+), 141 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1a707b99/src/kudu/client/client-test.cc
--
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index 28f12e2..6fad6e8 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -768,7 +768,7 @@ TEST_F(ClientTest, TestBadTable) {
   shared_ptr t;
   Status s = client_->OpenTable("xxx-does-not-exist", );
   ASSERT_TRUE(s.IsNotFound());
-  ASSERT_STR_CONTAINS(s.ToString(), "Not found: The table does not exist");
+  ASSERT_STR_CONTAINS(s.ToString(), "Not found: the table does not exist");
 }
 
 // Test that, if the master is down, we experience a network error talking
@@ -3926,7 +3926,7 @@ TEST_F(ClientTest, TestDeleteTable) {
   // Try to open the deleted table
   Status s = client_->OpenTable(kTableName, _table_);
   ASSERT_TRUE(s.IsNotFound());
-  ASSERT_STR_CONTAINS(s.ToString(), "The table does not exist");
+  ASSERT_STR_CONTAINS(s.ToString(), "the table does not exist");
 
   // Create a new table with the same name. This is to ensure that the client
   // doesn't cache anything inappropriately by table name (see KUDU-1055).
@@ -3946,7 +3946,7 @@ TEST_F(ClientTest, TestGetTableSchema) {
   // Verify that a get schema request for a missing table throws not found
   Status s = client_->GetTableSchema("MissingTableName", );
   ASSERT_TRUE(s.IsNotFound());
-  ASSERT_STR_CONTAINS(s.ToString(), "The table does not exist");
+  ASSERT_STR_CONTAINS(s.ToString(), "the table does not exist");
 }
 
 // Test creating and accessing a table which has multiple tablets,
@@ -4439,15 +4439,15 @@ TEST_F(ClientTest, TestCreateTableWithTooManyTablets) {
   .Create();
   ASSERT_TRUE(s.IsInvalidArgument());
   ASSERT_STR_CONTAINS(s.ToString(),
-  "The requested number of tablets is over the "
+  "the requested number of tablets is over the "
   "maximum permitted at creation time (1)");
 }
 
 // Tests for too many replicas, too few replicas, even replica count, etc.
 TEST_F(ClientTest, TestCreateTableWithBadNumReplicas) {
   const vector> cases = {
-{3, "Not enough live tablet servers to create a table with the requested "
- "replication factor 3. 1 tablet servers are alive"},
+{3, "not enough live tablet servers to create a table with the requested "
+ "replication factor 3; 1 tablet servers are alive"},
 {2, "illegal replication factor 2 (replication factor must be odd)"},
 {-1, "illegal replication factor -1 (replication factor must be 
positive)"},
 {11, "illegal replication factor 11 (max replication factor is 7)"}
@@ -4510,7 +4510,7 @@ TEST_F(ClientTest, TestCreateTable_TableNames) {
 // From 
http://stackoverflow.com/questions/1301402/example-invalid-utf8-string
 {string("foo\xf0\x28\x8c\xbc", 7), "invalid table name: invalid UTF8 
sequence"},
 // Should pass validation but fail due to lack of tablet servers running.
-{"你好", "Not enough live tablet servers"}
+{"你好", "not enough live tablet servers"}
   };
 
   for (const auto& test_case : kCases) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/1a707b99/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index bf87e4f..1de0be1 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -588,6 +588,16 @@ string RequestorString(RpcContext* rpc) {
   }
 }
 
+// If 's' is not OK, fills in 

kudu git commit: KUDU-2346: Fix mac pkg-config issue.

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/branch-1.7.x 5c68767d3 -> 5ef577b65


KUDU-2346: Fix mac pkg-config issue.

Previously an empty return string from pkg-config --cflags was used
as a proxy for testing if the package was installed. The MacOs version
of pkg-config returns an error message in response to this command
when a package is not installed.

The fix is to property test the return value of pkg-config --cflags.

Change-Id: I8a6b543dc65ac65d0dedf7e39f735b4805612bf1
Reviewed-on: http://gerrit.cloudera.org:8080/9652
Reviewed-by: Alexey Serbin 
Tested-by: Alexey Serbin 
(cherry picked from commit 9f6b7f16828250a89e0e331026eeb05687c6eeab)
Reviewed-on: http://gerrit.cloudera.org:8080/10373
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5ef577b6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5ef577b6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5ef577b6

Branch: refs/heads/branch-1.7.x
Commit: 5ef577b657f0e06b38cc2ce6e31f1b0b46eeab4d
Parents: 5c68767
Author: Attila Bukor 
Authored: Sat May 12 10:59:56 2018 +0200
Committer: Alexey Serbin 
Committed: Mon May 14 18:02:09 2018 +

--
 thirdparty/build-thirdparty.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/5ef577b6/thirdparty/build-thirdparty.sh
--
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index f91bc1f..beb8f97 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -171,8 +171,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
 
   # Build against the Macports or Homebrew OpenSSL versions, in order to match
   # the Kudu build.
-  OPENSSL_CFLAGS=$(pkg-config --cflags openssl)
-  if [ -z $OPENSSL_CFLAGS ]; then
+  if ! OPENSSL_CFLAGS=$(pkg-config --cflags openssl); then
 # If OpenSSL is built via Homebrew, pkg-config does not report on cflags.
 homebrew_openssl_dir=/usr/local/opt/openssl
 if [ -d $homebrew_openssl_dir ]; then

kudu git commit: [docs] Update macOS build docs

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master c18e7e1a6 -> 72df429ec


[docs] Update macOS build docs

* tapping homebrew/dupes is unnecessary now as it's now deprecated and
all formulae have been migrated to homebrew-core or deleted

Change-Id: Ic2a8aa7f58b7401115f36f42634d80b4d64e9794
Reviewed-on: http://gerrit.cloudera.org:8080/10356
Reviewed-by: Dan Burkert 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/72df429e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/72df429e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/72df429e

Branch: refs/heads/master
Commit: 72df429ecf95eaa01dee4743290395b650ec2ed4
Parents: c18e7e1
Author: Attila Bukor 
Authored: Fri May 11 18:16:29 2018 +0200
Committer: Dan Burkert 
Committed: Fri May 11 17:04:20 2018 +

--
 docs/installation.adoc | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/72df429e/docs/installation.adoc
--
diff --git a/docs/installation.adoc b/docs/installation.adoc
index 8d22655..236c325 100644
--- a/docs/installation.adoc
+++ b/docs/installation.adoc
@@ -551,7 +551,6 @@ for more information.
 . Install the prerequisite libraries, if they are not installed.
 +
 
-$ brew tap homebrew/dupes
 $ brew install autoconf automake cmake git krb5 libtool openssl pkg-config 
pstree
 
 

[2/3] kudu git commit: Fix int overflow GetClockTimeMicros() on macOS

[danburkert]

Fix int overflow GetClockTimeMicros() on macOS

On macOS mach_timespec_t.tv_sec is only 4 bytes and we
were converting to micros before moving to a bigger var.
This would cause all the wall times obtained on a mac (through
GetClockTimeMicros() to be wrong.

This was likely the cause of KUDU-2435 and KUDU-2408 too, since
the time would easily wrap, causing us to a update the clock with
a value that was seemingly from the future.

Posix just requires it to be an integer
(see: https://en.wikipedia.org/w/index.php?title=Time_t=450752800)
so also fixed it on the non-macOS path.

Testing this is likely not worth it since the only real change
was on macOS where the overlow doesn't happen anymore.

Change-Id: Ie0eaa548f61352be529755a732566613cfa72098
Reviewed-on: http://gerrit.cloudera.org:8080/10371
Reviewed-by: Todd Lipcon 
Reviewed-by: Dan Burkert 
Reviewed-by: Grant Henke 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/07d6b5f2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/07d6b5f2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/07d6b5f2

Branch: refs/heads/master
Commit: 07d6b5f2ba384e10288d5926af93933f0a74185d
Parents: 525943f
Author: David Alves 
Authored: Thu May 10 13:52:14 2018 -0700
Committer: David Ribeiro Alves 
Committed: Thu May 10 22:38:23 2018 +

--
 src/kudu/gutil/walltime.h | 15 +--
 1 file changed, 13 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/07d6b5f2/src/kudu/gutil/walltime.h
--
diff --git a/src/kudu/gutil/walltime.h b/src/kudu/gutil/walltime.h
index e9cab67..4b15f42 100644
--- a/src/kudu/gutil/walltime.h
+++ b/src/kudu/gutil/walltime.h
@@ -81,7 +81,12 @@ inline void GetCurrentTime(mach_timespec_t* ts) {
 inline MicrosecondsInt64 GetCurrentTimeMicros() {
   mach_timespec_t ts;
   GetCurrentTime();
-  return ts.tv_sec * 100 + ts.tv_nsec / 1000;
+  // 'tv_sec' is just 4 bytes on macOS, need to be careful not
+  // to convert to nanos until we've moved to a larger int.
+  MicrosecondsInt64 micros_from_secs = ts.tv_sec;
+  micros_from_secs *= 1000 * 1000;
+  micros_from_secs += ts.tv_nsec / 1000;
+  return micros_from_secs;
 }
 
 inline int64_t GetMonoTimeNanos() {
@@ -130,7 +135,13 @@ inline MicrosecondsInt64 GetThreadCpuTimeMicros() {
 inline MicrosecondsInt64 GetClockTimeMicros(clockid_t clock) {
   timespec ts;
   clock_gettime(clock, );
-  return ts.tv_sec * 100 + ts.tv_nsec / 1000;
+  // 'tv_sec' is usually 8 bytes, but the spec says it only
+  // needs to be 'a signed int'. Moved to a 64 bit var before
+  // converting to micros to be safe.
+  MicrosecondsInt64 micros_from_secs = ts.tv_sec;
+  micros_from_secs *= 1000 * 1000;
+  micros_from_secs += ts.tv_nsec / 1000;
+  return micros_from_secs;
 }
 
 #endif // defined(__APPLE__)

[1/3] kudu git commit: [tools] Add ids to tables and tablets; table name to tablets

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 3f43c03a1 -> 3855328b7


[tools] Add ids to tables and tablets; table name to tablets

This adds tablet id, table id, and table name to KsckTabletSummary,
and table id to KsckTableSummary.

This will be tested in the follow-up JSON formatting patch.

Change-Id: I19cd76a4c4c59c28930a44c0593021039903bec1
Reviewed-on: http://gerrit.cloudera.org:8080/10369
Reviewed-by: Alexey Serbin 
Tested-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/525943fd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/525943fd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/525943fd

Branch: refs/heads/master
Commit: 525943fd189acc20a3199f84431d62a776dd8f30
Parents: 3f43c03
Author: Will Berkeley 
Authored: Sun May 6 12:56:41 2018 -0700
Committer: Will Berkeley 
Committed: Thu May 10 20:07:41 2018 +

--
 src/kudu/tools/ksck-test.cc   |  5 +++--
 src/kudu/tools/ksck.cc|  6 +-
 src/kudu/tools/ksck.h | 12 ++--
 src/kudu/tools/ksck_remote.cc |  3 ++-
 src/kudu/tools/ksck_results.h |  4 
 5 files changed, 24 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck-test.cc
--
diff --git a/src/kudu/tools/ksck-test.cc b/src/kudu/tools/ksck-test.cc
index f4011cc..18d28ef 100644
--- a/src/kudu/tools/ksck-test.cc
+++ b/src/kudu/tools/ksck-test.cc
@@ -278,8 +278,9 @@ class KsckTest : public KuduTest {
 table->set_tablets({ tablet });
   }
 
-  shared_ptr CreateAndAddTable(const string& name, int 
num_replicas) {
-shared_ptr table(new KsckTable(name, Schema(), num_replicas));
+  shared_ptr CreateAndAddTable(const string& id_and_name, int 
num_replicas) {
+shared_ptr table(new KsckTable(id_and_name, id_and_name,
+  Schema(), num_replicas));
 cluster_->tables_.push_back(table);
 return table;
   }

http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck.cc
--
diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc
index 191838b..70a1b64 100644
--- a/src/kudu/tools/ksck.cc
+++ b/src/kudu/tools/ksck.cc
@@ -716,10 +716,11 @@ bool Ksck::VerifyTable(const shared_ptr& 
table) {
   }
 
   KsckTableSummary ts;
+  ts.id = table->id();
+  ts.name = table->name();
   ts.replication_factor = table->num_replicas();
   VLOG(1) << Substitute("Verifying $0 tablet(s) for table $1 configured with 
num_replicas = $2",
 tablets.size(), table->name(), table->num_replicas());
-  ts.name = table->name();
   for (const auto& tablet : tablets) {
 auto tablet_result = VerifyTablet(tablet, table->num_replicas());
 switch (tablet_result) {
@@ -876,6 +877,9 @@ KsckCheckResult Ksck::VerifyTablet(const 
shared_ptr& tablet,
   }
 
   KsckTabletSummary tablet_summary;
+  tablet_summary.id = tablet->id();
+  tablet_summary.table_id = tablet->table()->id();
+  tablet_summary.table_name = tablet->table()->name();
   tablet_summary.result = result;
   tablet_summary.status = status;
   tablet_summary.master_cstate = std::move(master_config);

http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck.h
--
diff --git a/src/kudu/tools/ksck.h b/src/kudu/tools/ksck.h
index edae81b..1c596e4 100644
--- a/src/kudu/tools/ksck.h
+++ b/src/kudu/tools/ksck.h
@@ -138,8 +138,15 @@ class KsckTablet {
 // Representation of a table. Composed of tablets.
 class KsckTable {
  public:
-  KsckTable(std::string name, const Schema& schema, int num_replicas)
-  : name_(std::move(name)), schema_(schema), num_replicas_(num_replicas) {}
+  KsckTable(std::string id, std::string name, const Schema& schema, int 
num_replicas)
+  : id_(std::move(id)),
+name_(std::move(name)),
+schema_(schema),
+num_replicas_(num_replicas) {}
+
+  const std::string& id() const {
+return id_;
+  }
 
   const std::string& name() const {
 return name_;
@@ -162,6 +169,7 @@ class KsckTable {
   }
 
  private:
+  const std::string id_;
   const std::string name_;
   const Schema schema_;
   const int num_replicas_;

http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck_remote.cc
--
diff --git a/src/kudu/tools/ksck_remote.cc b/src/kudu/tools/ksck_remote.cc
index d24f8c7..017321e 100644
--- a/src/kudu/tools/ksck_remote.cc
+++ b/src/kudu/tools/ksck_remote.cc
@@ -426,7 +426,8 @@ Status 

[3/3] kudu git commit: thirdparty: tweak clang compiler flags

[danburkert]

thirdparty: tweak clang compiler flags

Prior to this small tweak the thirdparty clang build output thousands of
warnings similar to this one when compiling with clang:

Building CXX object 
lib/Analysis/CMakeFiles/LLVMAnalysis.dir/LazyValueInfo.cpp.o
clang: warning: 
-Wl,-rpath,/Users/dan/src/cpp/kudu/thirdparty/installed/uninstrumented/lib: 
'linker' input unused [-Wunused-command-line-argument]
clang: warning: argument unused during compilation: 
'-L/Users/dan/src/cpp/kudu/thirdparty/installed/uninstrumented/lib' 
[-Wunused-command-line-argument]

Change-Id: Ide4ddbff14d3745c6f2c2f9b14b00da790a6cec6
Reviewed-on: http://gerrit.cloudera.org:8080/10352
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3855328b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3855328b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3855328b

Branch: refs/heads/master
Commit: 3855328b73c28d9ce4dbe5e1773bd6abde2e935a
Parents: 07d6b5f
Author: Dan Burkert 
Authored: Tue May 8 17:00:22 2018 -0700
Committer: Dan Burkert 
Committed: Thu May 10 22:53:24 2018 +

--
 thirdparty/build-definitions.sh | 29 -
 1 file changed, 16 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/3855328b/thirdparty/build-definitions.sh
--
diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh
index fe3f477..ebb4f72 100644
--- a/thirdparty/build-definitions.sh
+++ b/thirdparty/build-definitions.sh
@@ -252,6 +252,11 @@ build_llvm() {
 TOOLS_ARGS="$TOOLS_ARGS -D${arg}=OFF"
   done
 
+  # Remove '-nostdinc++' from the cflags for clang, since it already
+  # handles this when passing --stdlib=libc++ and passing this confuses
+  # the check for -fPIC.
+  CLANG_CXXFLAGS=$(echo "$EXTRA_CXXFLAGS" | sed -e 's,-nostdinc++,,g;')
+
   case $BUILD_TYPE in
 "normal")
   # Default build: core LLVM libraries, clang, compiler-rt, and all tools.
@@ -269,6 +274,15 @@ build_llvm() {
   if [ -n "$GCC_INSTALL_PREFIX" ]; then
 TOOLS_ARGS="$TOOLS_ARGS -DGCC_INSTALL_PREFIX=$GCC_INSTALL_PREFIX"
   fi
+
+  # Depend on zlib from the thirdparty tree. It's an optional dependency 
for
+  # LLVM, but a required [1] one for IWYU. When TSAN is enabled these flags
+  # are already set by build-thirdparty.sh in order to support the
+  # thirdparty libc++, so it's not necessary to set them again.
+  #
+  # 1. 
https://github.com/include-what-you-use/include-what-you-use/issues/539
+  CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include"
+  CLANG_LDFLAGS="$CLANG_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib"
   ;;
 "tsan")
   # Build just the core LLVM libraries, dependent on libc++.
@@ -303,18 +317,6 @@ build_llvm() {
  $PREFIX/lib/clang/ \
  $PREFIX/lib/cmake/{llvm,clang}
 
-  # Remove '-nostdinc++' from the cflags for clang, since it already
-  # handles this when passing --stdlib=libc++ and passing this confuses
-  # the check for -fPIC.
-  CLANG_CXXFLAGS=$(echo "$EXTRA_CXXFLAGS" | sed -e 's,-nostdinc++,,g;')
-
-  # Depend on zlib from the thirdparty tree. It's an optional dependency for
-  # LLVM, but a required [1] one for IWYU.
-  #
-  # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539
-  CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include"
-  CLANG_LDFLAGS="$EXTRA_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib"
-
   cmake \
 -DCMAKE_BUILD_TYPE=Release \
 -DCMAKE_INSTALL_PREFIX=$PREFIX \
@@ -324,7 +326,8 @@ build_llvm() {
 -DLLVM_INCLUDE_UTILS=OFF \
 -DLLVM_TARGETS_TO_BUILD=X86 \
 -DLLVM_ENABLE_RTTI=ON \
--DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS $CLANG_LDFLAGS" \
+-DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS $EXTRA_LDFLAGS" \
+-DCMAKE_EXE_LINKER_FLAGS="$CLANG_LDFLAGS" \
 -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
 $TOOLS_ARGS \
 $EXTRA_CMAKE_FLAGS \

kudu git commit: [thirdparty] Fix failing openssl check on MacOS

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master d83cd11a9 -> 60c8e305b


[thirdparty] Fix failing openssl check on MacOS

Since ea286417af the thirdparty build on Mac was failing as the openssl
library path needs to be set. This commit adds this flag by invoking
"pkg-config --cflags openssl" the same way as build-thirdparty.sh does.

Change-Id: Ieffc39f6e9bf5744b1119826375d36ee1035ccde
Reviewed-on: http://gerrit.cloudera.org:8080/10355
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/60c8e305
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/60c8e305
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/60c8e305

Branch: refs/heads/master
Commit: 60c8e305bef9ace8688aabbf18e2f500e8d9135c
Parents: d83cd11
Author: Attila Bukor 
Authored: Wed May 9 18:11:46 2018 +0200
Committer: Dan Burkert 
Committed: Wed May 9 16:50:08 2018 +

--
 thirdparty/preflight.py | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/60c8e305/thirdparty/preflight.py
--
diff --git a/thirdparty/preflight.py b/thirdparty/preflight.py
index cb77325..cf813de 100755
--- a/thirdparty/preflight.py
+++ b/thirdparty/preflight.py
@@ -132,6 +132,15 @@ def check_sasl():
   flags=["-E"]))
 
 def check_openssl():
+  cflags = ["-E"]
+  try:
+openssl_include = subprocess.check_output(["pkg-config", "--cflags", 
"openssl"]).rstrip()
+if openssl_include != "":
+  cflags.append(openssl_include)
+  except subprocess.CalledProcessError:
+  homebrew_openssl_dir="/usr/local/opt/openssl/include"
+  if os.path.isdir(homebrew_openssl_dir):
+cflags.append("-I" + homebrew_openssl_dir)
   try_do(
 "Checking for openssl headers",
 ("Unable to compile a simple program that uses openssl. " +
@@ -140,7 +149,7 @@ def check_openssl():
 lambda: compile("""
   #include 
   """,
-  flags=["-E"]))
+  flags=cflags))
 
 def main():
   print("Running pre-flight checks")

[1/2] kudu git commit: [Java] Fix TestSecurity tests on Mac

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 06afa8f04 -> eb4d88f0a


[Java] Fix TestSecurity tests on Mac

TestSecurity tests were failing because
InetAddress.getCanonicalHostName was returning
localhost for the 127.0.0.1 IP address which results
Kerberos rejecting as not kinit’d.

Other tests don’t have this issue because they extend
BaseKuduTest which installs the FakeDNS workaround
for 127.* IP addresses.

This patch adds the FakeDNS setup to the TestSecurity
class.

Change-Id: Ic61680f37c2d589ae4f5b553e69bc0b9eb15588d
Reviewed-on: http://gerrit.cloudera.org:8080/10298
Reviewed-by: Dan Burkert 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ff4b67e3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ff4b67e3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ff4b67e3

Branch: refs/heads/master
Commit: ff4b67e3b72f08d0e77d1803027ebaad965f6403
Parents: 06afa8f
Author: Grant Henke 
Authored: Thu May 3 15:42:26 2018 -0500
Committer: Grant Henke 
Committed: Thu May 3 21:40:58 2018 +

--
 .../src/test/java/org/apache/kudu/client/TestSecurity.java | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ff4b67e3/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
index 76e59cc..fc851e2 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java
@@ -36,6 +36,7 @@ import org.apache.kudu.util.SecurityUtil;
 import org.hamcrest.CoreMatchers;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import com.google.common.base.Preconditions;
@@ -83,6 +84,11 @@ public class TestSecurity {
 client.listTabletServers();
   }
 
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+FakeDNS.getInstance().install();
+  }
+
   @After
   public void reset() throws IOException, InterruptedException {
 if (client != null) {

[2/2] kudu git commit: mini-cluster: support parallel multi-master clusters

[danburkert]

mini-cluster: support parallel multi-master clusters

This commit refactors the mini-clusters to internally use reserved
sockets for their child daemons. Reserved sockets are simply sockets
bound to a random port with SO_REUSEPORT. As a result, master addresses
for multi-master mini clusters no longer need to hard-coded master
ports, and the associated ctest resource lock. This also significantly
lessens the chances that port conflicts will occur, although it is
still possible when masters are restarted during tests. To mitigate
that on Linux, tests are still run in a unique subnet.

Change-Id: I0b0ff7bfc179d8fdb1ed306d1bbd12acddeb060c
Reviewed-on: http://gerrit.cloudera.org:8080/8280
Reviewed-by: Adar Dembo 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eb4d88f0
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eb4d88f0
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eb4d88f0

Branch: refs/heads/master
Commit: eb4d88f0a376b13294c8985d76b3114eaa2e84ca
Parents: ff4b67e
Author: Dan Burkert 
Authored: Sun Oct 15 14:40:46 2017 -0700
Committer: Dan Burkert 
Committed: Thu May 3 21:43:01 2018 +

--
 src/kudu/client/client-test.cc  |   4 +-
 src/kudu/integration-tests/CMakeLists.txt   |  25 ++--
 .../catalog_manager_tsk-itest.cc|   1 -
 .../client-negotiation-failover-itest.cc|   2 -
 .../integration-tests/client-stress-test.cc |   1 -
 .../integration-tests/master-stress-test.cc |   3 +-
 .../master_cert_authority-itest.cc  |  16 +--
 .../integration-tests/master_failover-itest.cc  |   4 +-
 .../integration-tests/master_migration-itest.cc | 107 ---
 .../master_replication-itest.cc |  16 +--
 .../integration-tests/security-faults-itest.cc  |   1 -
 src/kudu/integration-tests/security-itest.cc|   1 -
 .../security-master-auth-itest.cc   |   4 +-
 .../integration-tests/token_signer-itest.cc |  23 ++--
 .../integration-tests/webserver-stress-itest.cc |   9 +-
 src/kudu/master/mini_master.cc  |   1 +
 src/kudu/mini-cluster/CMakeLists.txt|   2 +-
 .../mini-cluster/external_mini_cluster-test.cc  |   6 +-
 src/kudu/mini-cluster/external_mini_cluster.cc  | 119 ++---
 src/kudu/mini-cluster/external_mini_cluster.h   |  20 +--
 src/kudu/mini-cluster/internal_mini_cluster.cc  | 130 +--
 src/kudu/mini-cluster/internal_mini_cluster.h   |  26 +---
 src/kudu/mini-cluster/mini_cluster.cc   |  21 +++
 src/kudu/mini-cluster/mini_cluster.h|  19 ++-
 src/kudu/tools/CMakeLists.txt   |   2 +-
 src/kudu/tools/ksck_remote-test.cc  |   7 +-
 src/kudu/tools/tool_action_test.cc  |   3 -
 27 files changed, 248 insertions(+), 325 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/eb4d88f0/src/kudu/client/client-test.cc
--
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index 816fb39..28f12e2 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -774,7 +774,7 @@ TEST_F(ClientTest, TestBadTable) {
 // Test that, if the master is down, we experience a network error talking
 // to it (no "find the new leader master" since there's only one master).
 TEST_F(ClientTest, TestMasterDown) {
-  cluster_->mini_master()->Shutdown();
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
   shared_ptr t;
   client_->data_->default_admin_operation_timeout_ = MonoDelta::FromSeconds(1);
   Status s = client_->OpenTable("other-tablet", );
@@ -2643,7 +2643,7 @@ void 
ClientTest::DoTestWriteWithDeadServer(WhichServerToKill which) {
   // Shut down the server.
   switch (which) {
 case DEAD_MASTER:
-  cluster_->mini_master()->Shutdown();
+  cluster_->ShutdownNodes(cluster::ClusterNodes::MASTERS_ONLY);
   break;
 case DEAD_TSERVER:
   cluster_->mini_tablet_server(0)->Shutdown();

http://git-wip-us.apache.org/repos/asf/kudu/blob/eb4d88f0/src/kudu/integration-tests/CMakeLists.txt
--
diff --git a/src/kudu/integration-tests/CMakeLists.txt 
b/src/kudu/integration-tests/CMakeLists.txt
index f5497e2..60ff72a 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -53,7 +53,6 @@ add_dependencies(itest_util
 # Tests
 set(KUDU_TEST_LINK_LIBS itest_util ${KUDU_MIN_TEST_LIBS})
 ADD_KUDU_TEST(all_types-itest
-  RESOURCE_LOCK "master-rpc-ports"
   PROCESSORS 4
   NUM_SHARDS 8)
 ADD_KUDU_TEST(alter_table-randomized-test
@@ -62,10 

kudu git commit: KUDU-2191 (10/n): PoC: disallow Kudu alter column, create table operations from Hive

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 7767679a6 -> f54dbb891


KUDU-2191 (10/n): PoC: disallow Kudu alter column, create table operations from 
Hive

This commit makes it more difficult for processes or users who are not
the Kudu Master to alter the columns of Kudu table HMS entries through
Hive APIs.

The bulk of this patch is introducing a new environment context key/pair
('kudu.master_event=true') which is sent by the Kudu Master with every
HMS create/drop/alter table call. The KuduMetastorePlugin checks that
this property is set appropriately when operations modify a Kudu table
HMS entry.

The result is any attempt to create a table through Hive APIs or alter
the columns of a Kudu table through Hive APIs will fail. These
operations can not be supported because Hive lacks the DDL syntax and
HMS APIs to support necessary Kudu-specific table and column options.

It's possible for a user to spoof this environment context entry, but
it's a big enough roadblock that it should prevent accidental DDL
operations.

Change-Id: Idc39b1df07952d37e9ce93f1673aad535767a8ac
Reviewed-on: http://gerrit.cloudera.org:8080/10111
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f54dbb89
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f54dbb89
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f54dbb89

Branch: refs/heads/master
Commit: f54dbb89183db5559f7dcf76e86239e127f7f023
Parents: 7767679
Author: Dan Burkert 
Authored: Wed Apr 18 17:18:53 2018 -0700
Committer: Dan Burkert 
Committed: Thu May 3 20:31:04 2018 +

--
 .../hive/metastore/KuduMetastorePlugin.java | 34 +++
 .../hive/metastore/TestKuduMetastorePlugin.java | 44 
 src/kudu/hms/hms_catalog.cc | 19 ++---
 src/kudu/hms/hms_catalog.h  |  4 ++
 src/kudu/hms/hms_client-test.cc |  7 +++-
 src/kudu/hms/hms_client.cc  | 18 
 src/kudu/hms/hms_client.h   | 16 ---
 src/kudu/integration-tests/master_hms-itest.cc  | 20 +
 8 files changed, 125 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f54dbb89/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
--
diff --git 
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
 
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
index 4991f10..7a0a8f2 100644
--- 
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
+++ 
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
@@ -17,6 +17,8 @@
 
 package org.apache.kudu.hive.metastore;
 
+import java.util.Map;
+
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
@@ -27,6 +29,7 @@ import 
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
 import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
 import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
 
 /**
  * The {@code KuduMetastorePlugin} intercepts DDL operations on Kudu table 
entries
@@ -61,6 +64,8 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
   static final String KUDU_TABLE_ID_KEY = "kudu.table_id";
   @VisibleForTesting
   static final String KUDU_MASTER_ADDRS_KEY = "kudu.master_addresses";
+  @VisibleForTesting
+  static final String KUDU_MASTER_EVENT = "kudu.master_event";
 
   public KuduMetastorePlugin(Configuration config) {
 super(config);
@@ -78,6 +83,10 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
   return;
 }
 
+if (!isKuduMasterAction(tableEvent)) {
+  throw new MetaException("Kudu tables may not be created through Hive");
+}
+
 checkKuduProperties(table);
   }
 
@@ -94,6 +103,9 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
   return;
 }
 
+// The kudu.master_event property isn't checked, because the kudu.table_id
+// property already implies this event is coming from a Kudu Master.
+
 Table table = tableEvent.getTable();
 
 // Check that the table being dropped is a Kudu table.
@@ -130,6 +142,11 @@ public class KuduMetastorePlugin extends 
MetaStoreEventListener {
 if (!newTableId.equals(oldTableId)) {
   throw new MetaException("Kudu table ID does 

kudu git commit: rpc: add experimental rpc_reuseport flag

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 96fa80503 -> eded05723


rpc: add experimental rpc_reuseport flag

This option sets the SO_REUSEPORT socket option on a server's bound
socket. This socket option is supported on all platforms which Kudu
supports, except RHEL 6.4 and below.

The motivation is to use the option in the minicluster so that master
ports can be reserved by the mini cluster control processes and reused
by the master processes.

When rpc_reuseport is set on RHEL 6.4 the server fails to startup with
the following message (which is expected):

$ cat /etc/redhat-release
CentOS release 6.4 (Final)
$ ./kudu-master --fs-wal-dir=/tmp/kudu-master --logtostderr 
--unlock-experimental-flags --rpc-reuseport=true
...
F0501 14:56:30.672461  3271 master_main.cc:74] Check failed: _s.ok() Bad 
status: Network error: failed to set SO_REUSEPORT: Protocol not available 
(error 92)

Change-Id: I5d8ce9faa646fa2be554f5cfdf8b6ed0c48b496e
Reviewed-on: http://gerrit.cloudera.org:8080/8279
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eded0572
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eded0572
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eded0572

Branch: refs/heads/master
Commit: eded05723e6166b676074c5a548ac1278c7c4401
Parents: 96fa805
Author: Dan Burkert 
Authored: Sun Oct 15 12:16:30 2017 -0700
Committer: Dan Burkert 
Committed: Thu May 3 16:52:50 2018 +

--
 src/kudu/rpc/messenger.cc|  12 +++-
 src/kudu/rpc/messenger.h |   7 +++
 src/kudu/rpc/rpc-test.cc |   4 +-
 src/kudu/server/rpc_server.cc|   7 ++-
 src/kudu/server/rpc_server.h |   1 +
 src/kudu/server/server_base.cc   |   4 ++
 src/kudu/util/net/socket-test.cc |   2 +-
 src/kudu/util/net/socket.cc  | 114 +++---
 src/kudu/util/net/socket.h   |   7 +++
 9 files changed, 88 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/eded0572/src/kudu/rpc/messenger.cc
--
diff --git a/src/kudu/rpc/messenger.cc b/src/kudu/rpc/messenger.cc
index 67cd502..17ac0c5 100644
--- a/src/kudu/rpc/messenger.cc
+++ b/src/kudu/rpc/messenger.cc
@@ -81,7 +81,8 @@ MessengerBuilder::MessengerBuilder(std::string name)
   rpc_encryption_("optional"),
   rpc_tls_ciphers_(kudu::security::SecurityDefaults::kDefaultTlsCiphers),
   
rpc_tls_min_protocol_(kudu::security::SecurityDefaults::kDefaultTlsMinVersion),
-  enable_inbound_tls_(false) {
+  enable_inbound_tls_(false),
+  reuseport_(false) {
 }
 
 MessengerBuilder& MessengerBuilder::set_connection_keepalive_time(const 
MonoDelta ) {
@@ -178,6 +179,11 @@ MessengerBuilder& MessengerBuilder::enable_inbound_tls() {
   return *this;
 }
 
+MessengerBuilder& MessengerBuilder::set_reuseport() {
+  reuseport_ = true;
+  return *this;
+}
+
 Status MessengerBuilder::Build(shared_ptr *msgr) {
   // Initialize SASL library before we start making requests
   RETURN_NOT_OK(SaslInit(!keytab_file_.empty()));
@@ -311,6 +317,9 @@ Status Messenger::AddAcceptorPool(const Sockaddr 
_addr,
   Socket sock;
   RETURN_NOT_OK(sock.Init(0));
   RETURN_NOT_OK(sock.SetReuseAddr(true));
+  if (reuseport_) {
+RETURN_NOT_OK(sock.SetReusePort(true));
+  }
   RETURN_NOT_OK(sock.Bind(accept_addr));
   Sockaddr remote;
   RETURN_NOT_OK(sock.GetSocketAddress());
@@ -402,6 +411,7 @@ Messenger::Messenger(const MessengerBuilder )
 rpc_negotiation_timeout_ms_(bld.rpc_negotiation_timeout_ms_),
 sasl_proto_name_(bld.sasl_proto_name_),
 keytab_file_(bld.keytab_file_),
+reuseport_(bld.reuseport_),
 retain_self_(this) {
   for (int i = 0; i < bld.num_reactors_; i++) {
 reactors_.push_back(new Reactor(retain_self_, i, bld));

http://git-wip-us.apache.org/repos/asf/kudu/blob/eded0572/src/kudu/rpc/messenger.h
--
diff --git a/src/kudu/rpc/messenger.h b/src/kudu/rpc/messenger.h
index 3835cea..64a804b 100644
--- a/src/kudu/rpc/messenger.h
+++ b/src/kudu/rpc/messenger.h
@@ -163,6 +163,9 @@ class MessengerBuilder {
   // Configure the messenger to enable TLS encryption on inbound connections.
   MessengerBuilder& enable_inbound_tls();
 
+  // Configure the messenger to set the SO_REUSEPORT socket option.
+  MessengerBuilder& set_reuseport();
+
   Status Build(std::shared_ptr *msgr);
 
  private:
@@ -185,6 +188,7 @@ class MessengerBuilder {
   std::string rpc_private_key_password_cmd_;
   std::string keytab_file_;
   bool enable_inbound_tls_;
+  bool reuseport_;
 };
 
 // A Messenger is a container for the reactor threads which run event loops
@@ 

kudu git commit: KUDU-2351 Add IP/port for Recv() failure

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master abbde75e1 -> 1f380f279


KUDU-2351 Add IP/port for Recv() failure

Error messages like "Recv() got EOF from remote", "recv error:" and
"BlockingRecv error: failed to read from TLS socket" don't contain the
remote address, making it hard to locate the server the connection
failed to.

This change introduces adds the peer's IP address and port number to
these error messages.

Examples:

Recv() got EOF from remote 127.0.0.1:7050
recv error from 127.0.0.1:7050: Resource temporarily unavailable
BlockingRecv error: failed to read from TLS socket (remote: \
127.0.0.1:7050): Can't send after socket shutdown

Change-Id: I22436b13bb351b132e1c0b7159294dd0c980c2b3
Reviewed-on: http://gerrit.cloudera.org:8080/9818
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1f380f27
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1f380f27
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1f380f27

Branch: refs/heads/master
Commit: 1f380f27960021b267766fcefb6764e95e1d98dc
Parents: abbde75
Author: Attila Bukor 
Authored: Thu Apr 19 21:18:45 2018 +0200
Committer: Dan Burkert 
Committed: Thu Apr 19 20:07:16 2018 +

--
 src/kudu/security/tls_socket-test.cc | 30 +++
 src/kudu/security/tls_socket.cc  | 11 +++-
 src/kudu/util/CMakeLists.txt |  1 +
 src/kudu/util/net/socket-test.cc | 89 +++
 src/kudu/util/net/socket.cc  | 12 +++--
 5 files changed, 138 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1f380f27/src/kudu/security/tls_socket-test.cc
--
diff --git a/src/kudu/security/tls_socket-test.cc 
b/src/kudu/security/tls_socket-test.cc
index db4ff9b..001a206 100644
--- a/src/kudu/security/tls_socket-test.cc
+++ b/src/kudu/security/tls_socket-test.cc
@@ -45,6 +45,7 @@
 #include "kudu/util/random.h"
 #include "kudu/util/random_util.h"
 #include "kudu/util/scoped_cleanup.h"
+#include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
@@ -171,6 +172,8 @@ class EchoServer {
 SleepFor(MonoDelta::FromMilliseconds(10));
   }
 }
+
+CHECK_OK(listener_.Close());
   });
   }
 
@@ -212,6 +215,33 @@ class EchoServer {
 
 void handler(int /* signal */) {}
 
+TEST_F(TlsSocketTest, TestRecvFailure) {
+EchoServer server;
+server.Start();
+unique_ptr client_sock;
+NO_FATALS(ConnectClient(server.listen_addr(), _sock));
+unique_ptr buf(new uint8_t[kEchoChunkSize]);
+
+SleepFor(MonoDelta::FromMilliseconds(100));
+server.Stop();
+
+size_t nwritten;
+ASSERT_OK(client_sock->BlockingWrite(buf.get(), kEchoChunkSize, ,
+MonoTime::Now() + kTimeout));
+size_t nread;
+
+ASSERT_OK(client_sock->BlockingRecv(buf.get(), kEchoChunkSize, ,
+MonoTime::Now() + kTimeout));
+
+Status s = client_sock->BlockingRecv(buf.get(), kEchoChunkSize, ,
+MonoTime::Now() + kTimeout);
+
+ASSERT_TRUE(!s.ok());
+ASSERT_TRUE(s.IsNetworkError());
+ASSERT_STR_MATCHES(s.message().ToString(), "BlockingRecv error: failed to 
read from "
+   "TLS socket \\(remote: 
127.0.0.1:[0-9]+\\): ");
+}
+
 // Test for failures to handle EINTR during TLS connection
 // negotiation and data send/receive.
 TEST_F(TlsSocketTest, TestTlsSocketInterrupted) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/1f380f27/src/kudu/security/tls_socket.cc
--
diff --git a/src/kudu/security/tls_socket.cc b/src/kudu/security/tls_socket.cc
index 4fce907..355f04b 100644
--- a/src/kudu/security/tls_socket.cc
+++ b/src/kudu/security/tls_socket.cc
@@ -20,14 +20,17 @@
 #include 
 
 #include 
+#include 
 #include 
 
 #include 
 #include 
 
 #include "kudu/gutil/basictypes.h"
+#include "kudu/gutil/strings/substitute.h"
 #include "kudu/security/openssl_util.h"
 #include "kudu/util/errno.h"
+#include "kudu/util/net/sockaddr.h"
 #include "kudu/util/net/socket.h"
 
 namespace kudu {
@@ -104,20 +107,24 @@ Status TlsSocket::Writev(const struct ::iovec *iov, int 
iov_len, int64_t *nwritt
 
 Status TlsSocket::Recv(uint8_t *buf, int32_t amt, int32_t *nread) {
   SCOPED_OPENSSL_NO_PENDING_ERRORS;
-  const char* kErrString = "failed to read from TLS socket";
 
   CHECK(ssl_);
   errno = 0;
   int32_t bytes_read = SSL_read(ssl_.get(), buf, amt);
   int save_errno = errno;
   if (bytes_read <= 0) {
+Sockaddr remote;
+Socket::GetPeerAddress();
+std::string kErrString = 

[1/2] kudu git commit: Add krb5-server to SLES prerequisite libraries

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master bba339b11 -> c25b7cbd8


Add krb5-server to SLES prerequisite libraries

I was running a test with Kudu on SLES and realized that the MiniKDC
doesn't work due to 'kdb5_util' not being installed.

Running "zypper install krb5-server" fixes this.

Change-Id: I12f78066f9dabe60d63dcea03a1ba22da942ac0f
Reviewed-on: http://gerrit.cloudera.org:8080/9727
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins
Reviewed-by: Sailesh Mukil 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bf835b0e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bf835b0e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bf835b0e

Branch: refs/heads/master
Commit: bf835b0ea8fd28b083edb773c66217b2e0842985
Parents: bba339b
Author: Sailesh Mukil 
Authored: Tue Mar 20 12:04:35 2018 -0700
Committer: Todd Lipcon 
Committed: Wed Apr 18 21:36:13 2018 +

--
 docs/installation.adoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bf835b0e/docs/installation.adoc
--
diff --git a/docs/installation.adoc b/docs/installation.adoc
index b782c56..8d22655 100644
--- a/docs/installation.adoc
+++ b/docs/installation.adoc
@@ -459,7 +459,7 @@ make -j4
 
 $ sudo zypper install autoconf automake curl cyrus-sasl-devel \
   cyrus-sasl-gssapi flex gcc gcc-c++ gdb git java-1_8_0-openjdk-devel \
-  krb5-devel libtool lsb-release make ntp openssl-devel patch \
+  krb5-devel krb5-server libtool lsb-release make ntp openssl-devel patch \
   pkg-config python rsync unzip vim
 
 

[2/2] kudu git commit: [spark] KUDU-2371: Add KuduWriteOptions class and ignoreNull option

[danburkert]

[spark] KUDU-2371: Add KuduWriteOptions class and ignoreNull option

This patch adds the KuduWriteOptions class to allow configuration of
writes to the Kudu table when writing with Spark. This allows
extensibility via adding more fields in the future. The instance of
this class is passed to functions (insert/delete/upsert/update) in
KuduContext.

KuduWriteOptions is also supported in DefaultSource APIs. Clients can
set up write options in the parameters from SparkSQL.

This patch also adds the ignoreNull write option so that users can
upsert/update only non-Null columns and leave the rest of the columns
unchanged.

For example, this feature is useful when users use Spark streaming to
process JSON and upsert to Kudu, because missing column values from
JSON are set to NULL, resulting in some existing row values being
upserted to Null, which is not desired.

Change-Id: Ide908ea29f572849eca0ba850ee197c1b22a07c8
Reviewed-on: http://gerrit.cloudera.org:8080/9834
Reviewed-by: Dan Burkert 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c25b7cbd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c25b7cbd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c25b7cbd

Branch: refs/heads/master
Commit: c25b7cbd8d2c2746effc4b6866ea5764bcdef157
Parents: bf835b0
Author: fwang29 
Authored: Tue Mar 27 16:43:05 2018 -0700
Committer: Dan Burkert 
Committed: Wed Apr 18 23:32:00 2018 +

--
 .../apache/kudu/spark/kudu/DefaultSource.scala  |  18 ++-
 .../apache/kudu/spark/kudu/KuduContext.scala|  59 ++--
 .../kudu/spark/kudu/KuduWriteOptions.scala  |  35 +
 .../apache/kudu/spark/kudu/OperationType.scala  |   5 -
 .../kudu/spark/kudu/DefaultSourceTest.scala | 136 ++-
 .../apache/kudu/spark/kudu/TestContext.scala|  10 ++
 6 files changed, 237 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c25b7cbd/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
--
diff --git 
a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala 
b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 58beec7..761fd79 100644
--- 
a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ 
b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -51,6 +51,8 @@ class DefaultSource extends RelationProvider with 
CreatableRelationProvider
   val OPERATION = "kudu.operation"
   val FAULT_TOLERANT_SCANNER = "kudu.faultTolerantScan"
   val SCAN_LOCALITY = "kudu.scanLocality"
+  val IGNORE_NULL = "kudu.ignoreNull"
+  val IGNORE_DUPLICATE_ROW_ERRORS = "kudu.ignoreDuplicateRowErrors"
 
   def defaultMasterAddrs: String = 
InetAddress.getLocalHost.getCanonicalHostName
 
@@ -72,9 +74,13 @@ class DefaultSource extends RelationProvider with 
CreatableRelationProvider
 val faultTolerantScanner = 
Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
   .getOrElse(false)
 val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, 
"closest_replica"))
+val ignoreDuplicateRowErrors = 
Try(parameters(IGNORE_DUPLICATE_ROW_ERRORS).toBoolean).getOrElse(false) ||
+  Try(parameters(OPERATION) == "insert-ignore").getOrElse(false)
+val ignoreNull = Try(parameters.getOrElse(IGNORE_NULL, 
"false").toBoolean).getOrElse(false)
+val writeOptions = new KuduWriteOptions(ignoreDuplicateRowErrors, 
ignoreNull)
 
 new KuduRelation(tableName, kuduMaster, faultTolerantScanner,
-  scanLocality, operationType, None)(sqlContext)
+  scanLocality, operationType, None, writeOptions)(sqlContext)
   }
 
   /**
@@ -83,7 +89,7 @@ class DefaultSource extends RelationProvider with 
CreatableRelationProvider
 * @param sqlContext
 * @param mode Only Append mode is supported. It will upsert or insert data
 * to an existing table, depending on the upsert parameter
-* @param parameters Necessary parameters for kudu.table and kudu.master
+* @param parameters Necessary parameters for kudu.table, kudu.master, etc..
 * @param data Dataframe to save into kudu
 * @return returns populated base relation
 */
@@ -116,7 +122,7 @@ class DefaultSource extends RelationProvider with 
CreatableRelationProvider
   private def getOperationType(opParam: String): OperationType = {
 opParam.toLowerCase match {
   case "insert" => Insert
-  case "insert-ignore" => InsertIgnore
+  case "insert-ignore" => Insert
   case "upsert" => Upsert
   case "update" => Update
   case "delete" => Delete
@@ -144,6 +150,7 @@ class 

kudu git commit: HmsCatalog: fix bug in reconnection backoff logic

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 133ef7e9d -> 361ee99f0


HmsCatalog: fix bug in reconnection backoff logic

This fixes an accidentally shadowed Status variable. The effect of this
bug was that the HmsCatalog would not throttle HMS reconnect attempts.
The tests actually took advantage of this, I've had to update them to
try ops multiple times after an HMS restart.

No tests are included, but I've left a TODO about adding tests when
HmsCatalog stats are introduced.

Change-Id: I726283ec8332d16942c82c73492709de4323a43d
Reviewed-on: http://gerrit.cloudera.org:8080/10028
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao 
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/361ee99f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/361ee99f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/361ee99f

Branch: refs/heads/master
Commit: 361ee99f07cc5fa5219c384539eac3e4e003be08
Parents: 133ef7e
Author: Dan Burkert 
Authored: Wed Apr 11 12:13:24 2018 -0700
Committer: Dan Burkert 
Committed: Thu Apr 12 22:06:40 2018 +

--
 src/kudu/hms/hms_catalog-test.cc   | 10 +-
 src/kudu/hms/hms_catalog.cc|  2 +-
 src/kudu/integration-tests/master_hms-itest.cc | 37 -
 3 files changed, 31 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/361ee99f/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index af0244b..7326f4c 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -399,6 +399,10 @@ TEST_F(HmsCatalogTest, TestReconnect) {
   // Shutdown the HMS and try a few operations.
   ASSERT_OK(StopHms());
 
+  // TODO(dan): once we have HMS catalog stats, assert that repeated attempts
+  // while the HMS is unavailable results in a non-linear number of reconnect
+  // attempts.
+
   Status s = hms_catalog_->CreateTable(kTableId, "default.b", schema);
   EXPECT_TRUE(s.IsNetworkError()) << s.ToString();
 
@@ -407,7 +411,11 @@ TEST_F(HmsCatalogTest, TestReconnect) {
 
   // Start the HMS back up and ensure that the same operations succeed.
   ASSERT_OK(StartHms());
-  EXPECT_OK(hms_catalog_->CreateTable(kTableId, "default.d", schema));
+  ASSERT_EVENTUALLY([&] {
+// HmsCatalog throttles reconnections, so it's necessary to wait out the 
backoff.
+ASSERT_OK(hms_catalog_->CreateTable(kTableId, "default.d", schema));
+  });
+
   NO_FATALS(CheckTable(kHmsDatabase, "a", kTableId, schema));
   NO_FATALS(CheckTable(kHmsDatabase, "d", kTableId, schema));
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/361ee99f/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index a8b70a2..49048c7 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -365,7 +365,7 @@ Status HmsCatalog::Reconnect() {
 reconnect_idx_ = (reconnect_idx_ + 1) % hms_addresses_.size();
 
 hms_client_ = HmsClient(address, options);
-Status s = hms_client_.Start();
+s = hms_client_.Start();
 if (s.ok()) {
   VLOG(1) << "Connected to Hive Metastore " << address.ToString();
   return Status::OK();

http://git-wip-us.apache.org/repos/asf/kudu/blob/361ee99f/src/kudu/integration-tests/master_hms-itest.cc
--
diff --git a/src/kudu/integration-tests/master_hms-itest.cc 
b/src/kudu/integration-tests/master_hms-itest.cc
index 979d855..a6428aa 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -39,6 +39,7 @@
 #include "kudu/util/net/net_util.h" // IWYU pragma: keep
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
 
 namespace kudu {
 
@@ -91,12 +92,6 @@ class MasterHmsTest : public ExternalMiniClusterITestBase {
 return Status::OK();
   }
 
-  Status RestartHms() {
-RETURN_NOT_OK(StopHms());
-RETURN_NOT_OK(StartHms());
-return Status::OK();
-  }
-
   Status CreateDatabase(const string& database_name) {
 hive::Database db;
 db.name = database_name;
@@ -218,7 +213,10 @@ TEST_F(MasterHmsTest, TestCreateTable) {
 
   // Start the HMS and try again.
   ASSERT_OK(StartHms());
-  ASSERT_OK(CreateKuduTable(hms_database_name, "foo"));
+  ASSERT_EVENTUALLY([&] {
+// HmsCatalog throttles reconnections, so it's necessary to wait out the 
backoff.
+ASSERT_OK(CreateKuduTable(hms_database_name, "foo"));
+  });
   

kudu git commit: Add hms_client-test to the IWYU filter

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 03d1fcb14 -> c09a62961


Add hms_client-test to the IWYU filter

This is in keeping with the other HMS files that trigger many false
positives on the upstream gerrit. I've verified that the file does pass
IWYU on a local instance.

Change-Id: Icb281ec42ea2f7dafe2a3cf91a4e3a3d9ba5150b
Reviewed-on: http://gerrit.cloudera.org:8080/9991
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c09a6296
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c09a6296
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c09a6296

Branch: refs/heads/master
Commit: c09a6296181ecbb7d91687536f70fdbaac9fff5e
Parents: 03d1fcb
Author: Dan Burkert 
Authored: Wed Apr 11 09:31:38 2018 -0700
Committer: Dan Burkert 
Committed: Wed Apr 11 17:52:09 2018 +

--
 build-support/iwyu/iwyu-filter.awk |  1 +
 src/kudu/hms/hms_client-test.cc| 12 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c09a6296/build-support/iwyu/iwyu-filter.awk
--
diff --git a/build-support/iwyu/iwyu-filter.awk 
b/build-support/iwyu/iwyu-filter.awk
index 42f747f..0bd1034 100644
--- a/build-support/iwyu/iwyu-filter.awk
+++ b/build-support/iwyu/iwyu-filter.awk
@@ -92,6 +92,7 @@ BEGIN {
   muted["kudu/experiments/rwlock-perf.cc"]
   muted["kudu/hms/hms_catalog.cc"]
   muted["kudu/hms/hms_catalog.h"]
+  muted["kudu/hms/hms_client-test.cc"]
   muted["kudu/hms/hms_client.cc"]
   muted["kudu/hms/hms_client.h"]
   muted["kudu/hms/mini_hms.h"]

http://git-wip-us.apache.org/repos/asf/kudu/blob/c09a6296/src/kudu/hms/hms_client-test.cc
--
diff --git a/src/kudu/hms/hms_client-test.cc b/src/kudu/hms/hms_client-test.cc
index a4a6ffe..c308f90 100644
--- a/src/kudu/hms/hms_client-test.cc
+++ b/src/kudu/hms/hms_client-test.cc
@@ -17,18 +17,18 @@
 
 #include "kudu/hms/hms_client.h"
 
-#include  // IWYU pragma: keep
+#include 
 #include 
-#include  // IWYU pragma: keep
-#include  // IWYU pragma: keep
+#include 
+#include 
 #include 
-#include  // IWYU pragma: keep
+#include 
 #include 
 #include 
 
-#include  // IWYU pragma: keep
+#include 
 #include 
-#include  // IWYU pragma: keep
+#include 
 #include 
 
 #include "kudu/hms/hive_metastore_constants.h"

[2/2] kudu git commit: KUDU-2191 (9/n): HMS Catalog should short-circuit no-op alter and create table calls

[danburkert]

KUDU-2191 (9/n): HMS Catalog should short-circuit no-op alter and create table 
calls

This tweaks the HMS catalog class introduced in part 7 to be more
flexible with how tables are created and altered.  In particular, when
altering, renaming, or creating a table entry in the HMS, it returns an
OK status if the HMS already has the proper table entry, effectively
short-circuiting the operation. Some short-circuiting existed before,
but this makes it apply more uniformly in more situations.  This is
necessary for the notification listener to function correctly, since
it's not always possible to determine which notification log events have
aready been applied.

I also took the opportunity to refactor HMS catalog a bit, since
otherwise it would have been very unweildy to implement this
functionality. I've added some specific test cases to hms_catalog-test,
and some error message strings have changed slightly.

Change-Id: I9c3594db0db253bc88e032b8c5aa2c8667c49853
Reviewed-on: http://gerrit.cloudera.org:8080/9965
Reviewed-by: Adar Dembo 
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/00c2754c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/00c2754c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/00c2754c

Branch: refs/heads/master
Commit: 00c2754cae8bd58bf3fa61ea95da74eed4133149
Parents: 21f651d
Author: Dan Burkert 
Authored: Mon Apr 9 15:46:47 2018 -0700
Committer: Dan Burkert 
Committed: Tue Apr 10 18:55:45 2018 +

--
 src/kudu/hms/hms_catalog-test.cc   |  68 ++-
 src/kudu/hms/hms_catalog.cc| 123 
 src/kudu/hms/hms_catalog.h |  22 +++-
 src/kudu/hms/hms_client.cc |   1 +
 src/kudu/hms/hms_client.h  |   1 +
 src/kudu/integration-tests/master_hms-itest.cc |   4 +-
 6 files changed, 134 insertions(+), 85 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/00c2754c/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index d060320..af0244b 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -258,6 +258,17 @@ TEST_P(HmsCatalogTestParameterized, TestTableLifecycle) {
   ASSERT_OK(hms_catalog_->CreateTable(kTableId, kTableName, schema));
   NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, schema));
 
+  // Create the table again. This should succeed since the table ID matches. 
The
+  // HMS catalog will automatically short-circuit creating the table.
+  // TODO(dan): once we have HMS catalog stats, assert that the op short 
circuits.
+  ASSERT_OK(hms_catalog_->CreateTable(kTableId, kTableName, schema));
+  NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, schema));
+
+  // Create the table again, but with a different table ID.
+  Status s = hms_catalog_->CreateTable("new-table-id", kTableName, schema);
+  ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString();
+  NO_FATALS(CheckTable(kHmsDatabase, kHmsTableName, kTableId, schema));
+
   // Alter the table.
   SchemaBuilder b(schema);
   b.AddColumn("new_column", DataType::INT32);
@@ -305,70 +316,73 @@ TEST_F(HmsCatalogTest, TestLegacyTables) {
 // belong to external tables from other systems.
 TEST_F(HmsCatalogTest, TestExternalTable) {
   const string kTableId = "table-id";
-  const string kHmsDatabase = "default";
-
-  const string kHmsExternalTable = "external_table";
-  const string kExternalTableName = Substitute("$0.$1", kHmsDatabase, 
kHmsExternalTable);
-
-  const string kHmsKuduTable = "kudu_table";
-  const string kKuduTableName = Substitute("$0.$1", kHmsDatabase, 
kHmsKuduTable);
 
-  // Create the external table.
+  // Create the external table (default.ext).
   hive::Table external_table;
-  external_table.dbName = kHmsDatabase;
-  external_table.tableName = kHmsExternalTable;
+  external_table.dbName = "default";
+  external_table.tableName = "ext";
   external_table.tableType = HmsClient::kManagedTable;
   ASSERT_OK(hms_client_->CreateTable(external_table));
   // Retrieve the full HMS table object so it can be compared later (the HMS
   // fills in some fields during creation).
-  ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kHmsExternalTable, 
_table));
+  ASSERT_OK(hms_client_->GetTable("default", "ext", _table));
 
   auto CheckExternalTable = [&] {
 hive::Table current_table;
-ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kHmsExternalTable, 
_table));
+ASSERT_OK(hms_client_->GetTable("default", "ext", _table));
 ASSERT_EQ(current_table, external_table);
   };
 
-  

[1/2] kudu git commit: Add GetFlags endpoint and tool

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 337a731c4 -> 00c2754ca


 Add GetFlags endpoint and tool

This adds an rpc endpoint that retrieves gflags from servers.

It also includes a tool for retrieving flag values from servers.
By default, it returns only flags with non-default values. It
supports returning all flags, and also filtering flags by tags.

Example output from the tool run against a local cluster's master:

  flag  | value | 
default? |  tags
+---+--+-
 log_dir| /tmp/kudu/logs/master/0   | false 
   | stable
 heap_profile_path  | /tmp/kudu-master.56285| false 
   | advanced,stable
 log_filename   | kudu-master   | false 
   | stable
 webserver_interface| 127.0.0.1 | false 
   | advanced
 master_addresses   | 127.0.0.1:7053,127.0.0.1:7052,127.0.0.1:7051, | false 
   | stable
 fs_data_dirs   | /tmp/kudu/data/master/0/0 | false 
   | stable
 rpc_bind_addresses | 127.0.0.1:7051| false 
   | stable
 fs_wal_dir | /tmp/kudu/wal/master/0| false 
   | stable
 evict_failed_followers | false | false 
   | advanced
 webserver_port | 8051  | false 
   | stable

The rpc endpoint will also be used by ksck in a follow-up.

Change-Id: Ia35b4261099c1a3c6e2ff68e907c84df9a7ff699
Reviewed-on: http://gerrit.cloudera.org:8080/9948
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/21f651db
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/21f651db
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/21f651db

Branch: refs/heads/master
Commit: 21f651db1c51d80c9ce2761e3d07aa1129d1faca
Parents: 337a731
Author: Will Berkeley 
Authored: Mon Apr 2 16:13:21 2018 -0700
Committer: Will Berkeley 
Committed: Tue Apr 10 18:44:49 2018 +

--
 src/kudu/server/generic_service.cc | 36 ++-
 src/kudu/server/generic_service.h  |  6 +++
 src/kudu/server/server_base.proto  | 24 ++
 src/kudu/tools/kudu-tool-test.cc   | 48 
 src/kudu/tools/tool_action_common.cc   | 44 +++
 src/kudu/tools/tool_action_common.h|  5 +++
 src/kudu/tools/tool_action_master.cc   | 14 ++
 src/kudu/tools/tool_action_tserver.cc  | 14 ++
 src/kudu/tserver/tablet_server-test.cc | 68 +
 src/kudu/util/flags.cc | 26 +--
 src/kudu/util/flags.h  |  2 +
 11 files changed, 273 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/21f651db/src/kudu/server/generic_service.cc
--
diff --git a/src/kudu/server/generic_service.cc 
b/src/kudu/server/generic_service.cc
index e786c63..300e37f 100644
--- a/src/kudu/server/generic_service.cc
+++ b/src/kudu/server/generic_service.cc
@@ -17,9 +17,10 @@
 
 #include "kudu/server/generic_service.h"
 
-#include 
 #include 
+#include 
 #include 
+#include 
 
 #include 
 #include 
@@ -39,6 +40,7 @@
 #include "kudu/util/debug-util.h"
 #include "kudu/util/debug/leak_annotations.h" // IWYU pragma: keep
 #include "kudu/util/flag_tags.h"
+#include "kudu/util/flags.h"
 #include "kudu/util/status.h"
 
 DECLARE_string(time_source);
@@ -71,6 +73,38 @@ bool GenericServiceImpl::AuthorizeClient(const 
google::protobuf::Message* /*req*
 }
 
 
+void GenericServiceImpl::GetFlags(const GetFlagsRequestPB* req,
+  GetFlagsResponsePB* resp,
+  rpc::RpcContext* rpc) {
+  // If no tags were specified, return all flags that have non-default values.
+  // If tags were specified, also filter flags that don't match any tag.
+  bool all_flags = req->all_flags();
+  for (const auto& entry : GetFlagsMap()) {
+if (entry.second.is_default && !all_flags) {
+  continue;
+}
+unordered_set tags;
+GetFlagTags(entry.first, );
+bool matches = req->tags().empty();
+for (const auto& tag : req->tags()) {
+  if (ContainsKey(tags, tag)) {
+matches = true;
+break;
+  }
+}
+if (!matches) continue;
+
+auto* flag = resp->add_flags();
+flag->set_name(entry.first);
+flag->set_value(CheckFlagAndRedact(entry.second, EscapeMode::NONE));
+flag->set_is_default_value(entry.second.current_value == 

kudu git commit: KUDU-2401: External TLS certificate with Intermediate CA in server cert file fails

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master d1c99e02f -> 109fdc84e


KUDU-2401: External TLS certificate with Intermediate CA in server cert file 
fails

Take 2 certificate files: cert.pem and truststore.pem

cert.pem has 2 certificates in it:
A cert for that node (with CN="hostname", and signed by CN=CertToolkitIntCA)
And the intermediate CA cert (with CN=CertToolkitIntCA, and signed by 
CN=CertToolkitRootCA)

truststore.pem has 1 certificate in it:
A cert which is the root CA (with CN=CertToolkitRootCA, self-signed)

This previously would not work with KRPC because in
TlsContext::VerifyCertChainUnlocked(), we would only verify X509_verify_cert()
with the top certificate in the server certificate chain.

With this change, we pass the chain to X509_STORE_CTX_init() as well to make
sure that the entire chain gets checked against the CA.

A test is added that uses the specific certificate format mentioned above
and added to rpc-test.

TODO: Add a test case that has multiple intermediate CAs. Right now we're 
testing
with only one intermediate CA.

Change-Id: If4af35e97ec6f91c1d9ed902128bd7f4e260f0f4
Reviewed-on: http://gerrit.cloudera.org:8080/9934
Reviewed-by: Sailesh Mukil 
Reviewed-by: Dan Burkert 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/109fdc84
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/109fdc84
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/109fdc84

Branch: refs/heads/master
Commit: 109fdc84e45523fdbd8b1ca29110965187b10323
Parents: d1c99e0
Author: Sailesh Mukil 
Authored: Thu Apr 5 11:30:13 2018 -0700
Committer: Dan Burkert 
Committed: Fri Apr 6 00:23:00 2018 +

--
 src/kudu/rpc/rpc-test.cc |  35 -
 src/kudu/security/test/test_certs.cc | 219 ++
 src/kudu/security/test/test_certs.h  |   9 +-
 src/kudu/security/tls_context.cc |   4 +-
 4 files changed, 263 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/109fdc84/src/kudu/rpc/rpc-test.cc
--
diff --git a/src/kudu/rpc/rpc-test.cc b/src/kudu/rpc/rpc-test.cc
index 61a619f..89ea70d 100644
--- a/src/kudu/rpc/rpc-test.cc
+++ b/src/kudu/rpc/rpc-test.cc
@@ -187,7 +187,8 @@ TEST_P(TestRpc, TestCall) {
   }
 }
 
-TEST_P(TestRpc, TestCallWithChainCerts) {
+// Test for KUDU-2091 and KUDU-2220.
+TEST_P(TestRpc, TestCallWithChainCertAndChainCA) {
   bool enable_ssl = GetParam();
   // We're only interested in running this test with TLS enabled.
   if (!enable_ssl) return;
@@ -218,6 +219,38 @@ TEST_P(TestRpc, TestCallWithChainCerts) {
   ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
 }
 
+// Test for KUDU-2041.
+TEST_P(TestRpc, TestCallWithChainCertAndRootCA) {
+  bool enable_ssl = GetParam();
+  // We're only interested in running this test with TLS enabled.
+  if (!enable_ssl) return;
+
+  string rpc_certificate_file;
+  string rpc_private_key_file;
+  string rpc_ca_certificate_file;
+  
ASSERT_OK(security::CreateTestSSLCertWithChainSignedByRoot(GetTestDataDirectory(),
+ 
_certificate_file,
+ 
_private_key_file,
+ 
_ca_certificate_file));
+  // Set up server.
+  Sockaddr server_addr;
+  ASSERT_OK(StartTestServer(_addr, enable_ssl));
+
+  // Set up client.
+  SCOPED_TRACE(strings::Substitute("Connecting to $0", 
server_addr.ToString()));
+  shared_ptr client_messenger;
+  ASSERT_OK(CreateMessenger("Client", _messenger, 1, enable_ssl,
+  rpc_certificate_file, rpc_private_key_file, rpc_ca_certificate_file));
+
+  Proxy p(client_messenger, server_addr, server_addr.host(),
+  GenericCalculatorService::static_service_name());
+  ASSERT_STR_CONTAINS(p.ToString(), 
strings::Substitute("kudu.rpc.GenericCalculatorService@"
+"{remote=$0, 
user_credentials=",
+
server_addr.ToString()));
+
+  ASSERT_OK(DoTestSyncCall(p, GenericCalculatorService::kAddMethodName));
+}
+
 // Test making successful RPC calls while using a TLS certificate with a 
password protected
 // private key.
 TEST_P(TestRpc, TestCallWithPasswordProtectedKey) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/109fdc84/src/kudu/security/test/test_certs.cc
--
diff --git a/src/kudu/security/test/test_certs.cc 
b/src/kudu/security/test/test_certs.cc
index bc82140..88cf5cf 100644
--- 

kudu git commit: Followup to fe4d962c09012e to fix broken test

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 9e918bf77 -> de067f792


Followup to fe4d962c09012e to fix broken test

The newly introduced tests weren't cleaning up the test directory
because they didn't inherit from KuduTest.

Change-Id: I0e323f5c9082013510cddc6873bd81675e66d644
Reviewed-on: http://gerrit.cloudera.org:8080/9917
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/de067f79
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/de067f79
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/de067f79

Branch: refs/heads/master
Commit: de067f792afc051500a622b77d8c556511ae64f0
Parents: 9e918bf
Author: Dan Burkert 
Authored: Tue Apr 3 15:03:05 2018 -0700
Committer: Dan Burkert 
Committed: Tue Apr 3 23:00:47 2018 +

--
 src/kudu/hms/hms_catalog-test.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/de067f79/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 1510af0..d060320 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -37,6 +37,7 @@
 #include "kudu/util/net/net_util.h" // IWYU pragma: keep
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
 
 DECLARE_string(hive_metastore_uris);
 DECLARE_bool(hive_metastore_sasl_enabled);
@@ -113,7 +114,7 @@ TEST(HmsCatalogStaticTest, TestParseUris) {
 
 // Base class for HmsCatalog tests. Parameterized by whether
 // SASL/GSSAPI/Kerberos should be enabled.
-class HmsCatalogTest : public ::testing::Test {
+class HmsCatalogTest : public KuduTest {
  public:
 
   const char* const kMasterAddrs = "master-addrs";

kudu git commit: KUDU-2191 (8/n): Integrate HmsCatalog into CatalogManager

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 5fc31db00 -> 02560c188


KUDU-2191 (8/n): Integrate HmsCatalog into CatalogManager

This commit connects the CatalogManager to the HMS via the HmsCatalog
abstraction. When the HMS integration is enabled (by setting the
--hive-metastore-uris flag) and Kudu tables are created, altered, or
dropped, the corresponding HMS entry is also modified appropriately.
Additionally, New table and column names are required to be valid
according to the Hive identifier rules, which are much stricter than
Kudu's existing identifier rules.

Testing: This commit adds a new integration test (master_hms-itest)
which tests that the integration works as expected with
create/alter/drop table operations. Additionally, some existing DDL
stress tests now have the HMS integration enabled in order to provide
more coverage.

Change-Id: Ie68e143c3c317c7690af097e6485934feb1010b4
Reviewed-on: http://gerrit.cloudera.org:8080/9863
Reviewed-by: Dan Burkert 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/02560c18
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/02560c18
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/02560c18

Branch: refs/heads/master
Commit: 02560c188f6b76714ee1908420e539f28bce99e8
Parents: 5fc31db
Author: Dan Burkert 
Authored: Wed Mar 28 16:45:35 2018 -0700
Committer: Dan Burkert 
Committed: Tue Apr 3 19:23:58 2018 +

--
 build-support/iwyu/iwyu-filter.awk  |   6 +
 src/kudu/hms/hms_catalog.cc |  62 +--
 src/kudu/hms/hms_catalog.h  |  11 +-
 src/kudu/hms/hms_client.h   |   6 +-
 src/kudu/hms/mini_hms.h |   2 +-
 src/kudu/integration-tests/CMakeLists.txt   |   1 +
 .../alter_table-randomized-test.cc  |   3 +-
 .../integration-tests/master-stress-test.cc |   7 +-
 .../integration-tests/master_failover-itest.cc  |  43 +-
 src/kudu/integration-tests/master_hms-itest.cc  | 394 +++
 src/kudu/master/CMakeLists.txt  |   3 +-
 src/kudu/master/catalog_manager.cc  | 184 +++--
 src/kudu/master/catalog_manager.h   |   9 +-
 src/kudu/master/master.proto|   3 +
 src/kudu/mini-cluster/external_mini_cluster.cc  |  10 +
 15 files changed, 655 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/02560c18/build-support/iwyu/iwyu-filter.awk
--
diff --git a/build-support/iwyu/iwyu-filter.awk 
b/build-support/iwyu/iwyu-filter.awk
index c20f2d1..42f747f 100644
--- a/build-support/iwyu/iwyu-filter.awk
+++ b/build-support/iwyu/iwyu-filter.awk
@@ -90,7 +90,13 @@ BEGIN {
   muted["kudu/common/encoded_key-test.cc"]
   muted["kudu/common/schema.h"]
   muted["kudu/experiments/rwlock-perf.cc"]
+  muted["kudu/hms/hms_catalog.cc"]
+  muted["kudu/hms/hms_catalog.h"]
   muted["kudu/hms/hms_client.cc"]
+  muted["kudu/hms/hms_client.h"]
+  muted["kudu/hms/mini_hms.h"]
+  muted["kudu/master/catalog_manager.cc"]
+  muted["kudu/master/catalog_manager.h"]
   muted["kudu/rpc/reactor.cc"]
   muted["kudu/rpc/reactor.h"]
   muted["kudu/security/ca/cert_management.cc"]

http://git-wip-us.apache.org/repos/asf/kudu/blob/02560c18/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index 3cfbd90..a6433b3 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -17,13 +17,13 @@
 
 #include "kudu/hms/hms_catalog.h"
 
-#include  // IWYU pragma: keep
-#include  // IWYU pragma: keep
+#include 
+#include 
 #include 
-#include  // IWYU pragma: keep
+#include 
 #include 
-#include  // IWYU pragma: keep
-#include  // IWYU pragma: keep
+#include 
+#include 
 #include 
 
 #include 
@@ -33,16 +33,16 @@
 #include "kudu/common/schema.h"
 #include "kudu/common/types.h"
 #include "kudu/gutil/macros.h"
-#include "kudu/gutil/strings/split.h" // IWYU pragma: keep
+#include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/hms/hive_metastore_constants.h"
 #include "kudu/hms/hive_metastore_types.h"
 #include "kudu/hms/hms_client.h"
 #include "kudu/util/async_util.h"
 #include "kudu/util/flag_tags.h"
-#include "kudu/util/net/net_util.h" // IWYU pragma: keep
+#include "kudu/util/net/net_util.h"
 #include "kudu/util/slice.h"
-#include "kudu/util/threadpool.h" // IWYU pragma: keep
+#include "kudu/util/threadpool.h"
 
 using std::string;
 using std::vector;
@@ -273,8 +273,8 @@ Status HmsCatalog::AlterTable(const string& id,
 
 template
 Status 

kudu git commit: Followup to bae6170c77 to fix build on macos

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master fff4185e0 -> 96951f68f


Followup to bae6170c77 to fix build on macos

RLIM_INFINITY is kint64max on MacOS and kuint64max on Linux.

Change-Id: I13780d86d4a286710c1bcf8a5f4266fb7fe162c7
Reviewed-on: http://gerrit.cloudera.org:8080/9888
Reviewed-by: Adar Dembo 
Tested-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/96951f68
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/96951f68
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/96951f68

Branch: refs/heads/master
Commit: 96951f68f53fc44e97de99c2c7dfb036a59bd9fb
Parents: fff4185
Author: Dan Burkert 
Authored: Mon Apr 2 11:41:19 2018 -0700
Committer: Dan Burkert 
Committed: Mon Apr 2 22:07:03 2018 +

--
 src/kudu/util/env.h| 3 +--
 src/kudu/util/env_posix.cc | 5 ++---
 2 files changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/96951f68/src/kudu/util/env.h
--
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index 5cf357e..3e5981b 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -319,8 +319,7 @@ class Env {
 RUNNING_THREADS_PER_EUID,
   };
 
-  // Gets the process' current limit for the given resource type. If there is
-  // no limit, returns kuint64max.
+  // Gets the process' current limit for the given resource type.
   //
   // On UNIX platforms, this is equivalent to the resource's soft limit.
   virtual uint64_t GetResourceLimit(ResourceLimitType t) = 0;

http://git-wip-us.apache.org/repos/asf/kudu/blob/96951f68/src/kudu/util/env_posix.cc
--
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index d180cb3..46d72d2 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -39,7 +39,6 @@
 #include "kudu/gutil/bind.h"
 #include "kudu/gutil/bind_helpers.h"
 #include "kudu/gutil/gscoped_ptr.h"
-#include "kudu/gutil/integral_types.h"
 #include "kudu/gutil/macros.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/once.h"
@@ -1597,8 +1596,8 @@ class PosixEnv : public Env {
   }
 
   virtual uint64_t GetResourceLimit(ResourceLimitType t) OVERRIDE {
-static_assert(RLIM_INFINITY == kuint64max,
-  "RLIM_INFINITY is assumed to be 2^64 - 1");
+static_assert(std::is_unsigned::value, "rlim_t must be unsigned");
+static_assert(RLIM_INFINITY > 0, "RLIM_INFINITY must be positive");
 
 // There's no reason for this to ever fail.
 struct rlimit l;

[3/3] kudu git commit: KUDU-2191 (6/n): Fixups to the C++ HMS client

[danburkert]

KUDU-2191 (6/n): Fixups to the C++ HMS client

These are changes necessary for subsequent patches in the series, but
they are easier to review in isolation.

Change-Id: I86a8d984e67fe6b4c004725828b4296476c2389e
Reviewed-on: http://gerrit.cloudera.org:8080/9861
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fff4185e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fff4185e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fff4185e

Branch: refs/heads/master
Commit: fff4185e0f88954c67841a13b0924db4eab4c761
Parents: 4db748f
Author: Dan Burkert 
Authored: Wed Mar 28 16:35:45 2018 -0700
Committer: Dan Burkert 
Committed: Mon Apr 2 21:39:49 2018 +

--
 .../hive/metastore/TestKuduMetastorePlugin.java |  3 +-
 src/kudu/hms/hms_client-test.cc | 15 +--
 src/kudu/hms/hms_client.cc  | 44 ++--
 src/kudu/hms/hms_client.h   | 18 +++-
 src/kudu/hms/mini_hms.cc| 10 -
 5 files changed, 79 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
--
diff --git 
a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
 
b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
index 8eb6927..1abcd6a 100644
--- 
a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
+++ 
b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.MockPartitionExpressionForMetastore;
 import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
+import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -77,7 +78,7 @@ public class TestKuduMetastorePlugin {
 Table table = new Table();
 table.setDbName("default");
 table.setTableName(name);
-table.setTableType("MANAGED_TABLE");
+table.setTableType(TableType.MANAGED_TABLE.toString());
 table.putToParameters(hive_metastoreConstants.META_TABLE_STORAGE,
   KuduMetastorePlugin.KUDU_STORAGE_HANDLER);
 table.putToParameters(KuduMetastorePlugin.KUDU_TABLE_ID_KEY,

http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/src/kudu/hms/hms_client-test.cc
--
diff --git a/src/kudu/hms/hms_client-test.cc b/src/kudu/hms/hms_client-test.cc
index 87601eb..a4a6ffe 100644
--- a/src/kudu/hms/hms_client-test.cc
+++ b/src/kudu/hms/hms_client-test.cc
@@ -17,11 +17,16 @@
 
 #include "kudu/hms/hms_client.h"
 
+#include  // IWYU pragma: keep
 #include 
+#include  // IWYU pragma: keep
+#include  // IWYU pragma: keep
 #include 
+#include  // IWYU pragma: keep
 #include 
 #include 
 
+#include  // IWYU pragma: keep
 #include 
 #include  // IWYU pragma: keep
 #include 
@@ -59,7 +64,7 @@ class HmsClientTest : public KuduTest,
 hive::Table table;
 table.dbName = database_name;
 table.tableName = table_name;
-table.tableType = "MANAGED_TABLE";
+table.tableType = HmsClient::kManagedTable;
 
 table.__set_parameters({
 make_pair(HmsClient::kKuduTableIdKey, table_id),
@@ -122,7 +127,9 @@ TEST_P(HmsClientTest, TestHmsOperations) {
   ASSERT_OK(hms.Start());
 
   HmsClient client(hms.address(), hms_client_opts);
+  ASSERT_FALSE(client.IsConnected());
   ASSERT_OK(client.Start());
+  ASSERT_TRUE(client.IsConnected());
 
   // Create a database.
   string database_name = "my_db";
@@ -161,7 +168,7 @@ TEST_P(HmsClientTest, TestHmsOperations) {
   EXPECT_EQ(table_id, my_table.parameters[HmsClient::kKuduTableIdKey]);
   EXPECT_EQ(HmsClient::kKuduStorageHandler,
 
my_table.parameters[hive::g_hive_metastore_constants.META_TABLE_STORAGE]);
-  EXPECT_EQ("MANAGED_TABLE", my_table.tableType);
+  EXPECT_EQ(HmsClient::kManagedTable, my_table.tableType);
 
   string new_table_name = "my_altered_table";
 
@@ -186,7 +193,7 @@ TEST_P(HmsClientTest, TestHmsOperations) {
   EXPECT_EQ(table_id, renamed_table.parameters[HmsClient::kKuduTableIdKey]);
   EXPECT_EQ(HmsClient::kKuduStorageHandler,
 
renamed_table.parameters[hive::g_hive_metastore_constants.META_TABLE_STORAGE]);
- 

[2/3] kudu git commit: KUDU-2384. Don't collect thread stacks when running under debugger

[danburkert]

KUDU-2384. Don't collect thread stacks when running under debugger

This changes the stack watchdog and the 'collect all stacks' utility
code to not run when it appears that the process is running under a
tracer such as gdb or strace. This makes debugging Kudu processes
significantly easier because you don't need to know the magical
incantation 'handle SIGUSR2 nostop noprint'.

I tested manually by running a master with
--diagnostics-log-stack-traces-interval-ms=10 and attaching to it with
gdb. With this patch, I didn't see any signals sent.

This patch also changes the Env EIO fault injection to never inject
failures on /proc/. This is to avoid a TSAN warning since
KernelStackWatchdog would otherwise access the env_inject_eio_globs flag
concurrent to tests modifying it.

Change-Id: Ib17ffb9514e9b440b8d97843374d2a60913a398b
Reviewed-on: http://gerrit.cloudera.org:8080/9835
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4db748f3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4db748f3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4db748f3

Branch: refs/heads/master
Commit: 4db748f3021711a67017fe51f30097350f343101
Parents: 357ef6c
Author: Todd Lipcon 
Authored: Tue Mar 27 16:58:37 2018 -0700
Committer: Todd Lipcon 
Committed: Mon Apr 2 05:04:29 2018 +

--
 src/kudu/util/debug-util.cc|  5 +
 src/kudu/util/debug-util.h | 18 ---
 src/kudu/util/env_posix.cc | 17 +++---
 src/kudu/util/kernel_stack_watchdog.cc |  7 ++
 src/kudu/util/os-util.cc   | 35 +
 src/kudu/util/os-util.h|  7 +-
 6 files changed, 82 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/debug-util.cc
--
diff --git a/src/kudu/util/debug-util.cc b/src/kudu/util/debug-util.cc
index 07703ea..93a9543 100644
--- a/src/kudu/util/debug-util.cc
+++ b/src/kudu/util/debug-util.cc
@@ -67,6 +67,7 @@
 #include "kudu/util/errno.h"
 #include "kudu/util/faststring.h"
 #include "kudu/util/monotime.h"
+#include "kudu/util/os-util.h"
 #include "kudu/util/scoped_cleanup.h"
 #include "kudu/util/thread.h"
 
@@ -724,6 +725,10 @@ bool StackTrace::LessThan(const StackTrace& s) const {
 }
 
 Status StackTraceSnapshot::SnapshotAllStacks() {
+  if (IsBeingDebugged()) {
+return Status::Incomplete("not collecting stack trace since debugger or 
strace is attached");
+  }
+
   vector tids;
   RETURN_NOT_OK_PREPEND(ListThreads(), "could not list threads");
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/debug-util.h
--
diff --git a/src/kudu/util/debug-util.h b/src/kudu/util/debug-util.h
index 44fccc3..e8c94ea 100644
--- a/src/kudu/util/debug-util.h
+++ b/src/kudu/util/debug-util.h
@@ -68,10 +68,20 @@ Status SetStackTraceSignal(int signum);
 // thread. It requires that the target thread has not blocked POSIX signals. If
 // it has, an error message will be returned.
 //
-// This function is thread-safe but coarsely synchronized: only one "dumper" 
thread
-// may be active at a time.
+// This function is thread-safe.
+//
+// NOTE: if Kudu is running inside a debugger, this can be annoying to a 
developer since
+// it internally uses signals that will cause the debugger to stop. Consider 
checking
+// 'IsBeingDebugged()' from os-util.h before using this function for 
non-critical use
+// cases.
 std::string DumpThreadStack(int64_t tid);
 
+// Capture the thread stack of another thread
+//
+// NOTE: if Kudu is running inside a debugger, this can be annoying to a 
developer since
+// it internally uses signals that will cause the debugger to stop. Consider 
checking
+// 'IsBeingDebugged()' from os-util.h before using this function for 
non-critical use
+// cases.
 Status GetThreadStack(int64_t tid, StackTrace* stack);
 
 // Return the current stack trace, stringified.
@@ -230,7 +240,9 @@ class StackTraceSnapshot {
 capture_thread_names_ = c;
   }
 
-  // Snapshot the stack traces of all threads in the process.
+  // Snapshot the stack traces of all threads in the process. This may return 
a bad
+  // Status in the case that stack traces aren't supported on the platform, or 
if
+  // the process is running inside a debugger.
   //
   // NOTE: this may take some time and should not be called in a 
latency-sensitive
   // context.

http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/env_posix.cc
--
diff --git 

[1/3] kudu git commit: [examples] Add licenses to Python examples

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master ba955733d -> fff4185e0


[examples] Add licenses to Python examples

This also adds a README to the graphite example warning that it's
not a real integration, just an example.

Change-Id: I8825bfac0416fc19fdb99c972eb104898ba78e7d
Reviewed-on: http://gerrit.cloudera.org:8080/9858
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/357ef6c9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/357ef6c9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/357ef6c9

Branch: refs/heads/master
Commit: 357ef6c9f736a77ae1cec786088adff4d8ac5091
Parents: ba95573
Author: Will Berkeley 
Authored: Thu Mar 29 14:34:26 2018 -0700
Committer: Will Berkeley 
Committed: Mon Apr 2 04:26:44 2018 +

--
 examples/python/dstat-kudu/README.md| 14 
 examples/python/dstat-kudu/kudu_dstat.py| 17 ++
 examples/python/graphite-kudu/README.adoc   | 24 
 examples/python/graphite-kudu/kudu/__init__.py  | 17 ++
 .../python/graphite-kudu/kudu/kudu_graphite.py  | 19 +++-
 examples/python/graphite-kudu/setup.py  | 18 ++-
 python/setup.cfg|  2 +-
 7 files changed, 108 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/dstat-kudu/README.md
--
diff --git a/examples/python/dstat-kudu/README.md 
b/examples/python/dstat-kudu/README.md
index a4f443c..f2ba031 100644
--- a/examples/python/dstat-kudu/README.md
+++ b/examples/python/dstat-kudu/README.md
@@ -1,3 +1,17 @@
+
+
 # Kudu + dstat + Impala
 
 This is an example program that shows how to use the Kudu API in Python to 
load data into

http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/dstat-kudu/kudu_dstat.py
--
diff --git a/examples/python/dstat-kudu/kudu_dstat.py 
b/examples/python/dstat-kudu/kudu_dstat.py
index f1f5bb2..4bc3485 100644
--- a/examples/python/dstat-kudu/kudu_dstat.py
+++ b/examples/python/dstat-kudu/kudu_dstat.py
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 import kudu
 import os
 import subprocess

http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/graphite-kudu/README.adoc
--
diff --git a/examples/python/graphite-kudu/README.adoc 
b/examples/python/graphite-kudu/README.adoc
new file mode 100644
index 000..eede0d4
--- /dev/null
+++ b/examples/python/graphite-kudu/README.adoc
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+= Kudu Python client Graphite example
+:author: Kudu Team
+:homepage: https://kudu.apache.org/
+
+This uses the Python client to implement a plugin for using graphite-web with
+Kudu as a backend. Be aware it's meant just as an example of how to use the
+Python client. It is not a well-tested integration.

http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/graphite-kudu/kudu/__init__.py

kudu git commit: KUDU-2259: add real user to AuthenticationCredentialsPB

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 30cf90028 -> e684de337


KUDU-2259: add real user to AuthenticationCredentialsPB

This commit adds the 'real user' to the authn credentials token, which
is used when negotiating connections with SASL PLAIN authentication.
This is useful when scan tokens are being sent to remote tasks, it's not
possible to authenticate with a signed authn token to the remote
server[1], coarse-grained ACLs have been set, and the 'planner' and
'executor' processes are being run with different users.

This problematic scenario might also have been solved by allowing tokens
to be used in all scenarios, even when encryption is disabled, but the
approach taken by this commit allows that invariant to remain.

[1]: this most often occurs because the remote server has encryption disabled.

Change-Id: I5d2d901d42501ecfc0f6372f68cf7335eb188b45
Reviewed-on: http://gerrit.cloudera.org:8080/9374
Reviewed-by: Todd Lipcon 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e684de33
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e684de33
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e684de33

Branch: refs/heads/master
Commit: e684de3371941cc5ae8fc4a546ecda7dbe9f4f2f
Parents: 30cf900
Author: Dan Burkert 
Authored: Tue Feb 20 17:37:17 2018 -0800
Committer: Dan Burkert 
Committed: Thu Mar 15 02:47:42 2018 +

--
 .../java/org/apache/kudu/client/Negotiator.java |  8 +-
 .../org/apache/kudu/client/SecurityContext.java | 47 ++
 .../client/TestSecurityContextRealUser.java | 97 
 src/kudu/client/client-internal.cc  |  2 +
 src/kudu/client/client-internal.h   |  5 +
 src/kudu/client/client-test.cc  | 36 
 src/kudu/client/client.cc   | 23 -
 src/kudu/client/client.proto|  7 +-
 src/kudu/client/master_rpc.cc   | 10 ++
 src/kudu/client/master_rpc.h|  5 +
 src/kudu/client/meta_cache.cc   |  1 +
 src/kudu/mini-cluster/external_mini_cluster.cc  |  6 +-
 src/kudu/rpc/connection_id.cc   |  1 +
 src/kudu/rpc/server_negotiation.cc  |  6 +-
 src/kudu/rpc/user_credentials.cc|  7 +-
 src/kudu/rpc/user_credentials.h |  5 +
 src/kudu/util/user.cc   |  2 +-
 17 files changed, 235 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/e684de33/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
--
diff --git 
a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java 
b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
index 95ad907..47e4854 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
@@ -92,7 +92,7 @@ import org.apache.kudu.util.SecurityUtil;
 public class Negotiator extends SimpleChannelUpstreamHandler {
   private static final Logger LOG = LoggerFactory.getLogger(Negotiator.class);
 
-  private static final SaslClientCallbackHandler SASL_CALLBACK = new 
SaslClientCallbackHandler();
+  private final SaslClientCallbackHandler SASL_CALLBACK = new 
SaslClientCallbackHandler();
   private static final Set SUPPORTED_RPC_FEATURES =
   ImmutableSet.of(
   RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS,
@@ -772,7 +772,7 @@ public class Negotiator extends 
SimpleChannelUpstreamHandler {
 
 // The UserInformationPB is deprecated, but used by servers prior to Kudu 
1.1.
 RpcHeader.UserInformationPB.Builder userBuilder = 
RpcHeader.UserInformationPB.newBuilder();
-String user = System.getProperty("user.name");
+String user = securityContext.getRealUser();
 userBuilder.setEffectiveUser(user);
 userBuilder.setRealUser(user);
 builder.setDEPRECATEDUserInfo(userBuilder.build());
@@ -826,11 +826,11 @@ public class Negotiator extends 
SimpleChannelUpstreamHandler {
 }
   }
 
-  private static class SaslClientCallbackHandler implements CallbackHandler {
+  private class SaslClientCallbackHandler implements CallbackHandler {
 public void handle(Callback[] callbacks) throws 
UnsupportedCallbackException {
   for (Callback callback : callbacks) {
 if (callback instanceof NameCallback) {
-  ((NameCallback) callback).setName(System.getProperty("user.name"));
+  ((NameCallback) callback).setName(securityContext.getRealUser());
 } else if (callback instanceof PasswordCallback) {
   ((PasswordCallback) 

kudu git commit: Remove namespace hack in hms_client.h

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master debcb8ea2 -> b525378c5


Remove namespace hack in hms_client.h

To avoid having to use the very long Apache.Hadoop.Hive namespace for
all of the Thrift-generated HMS types, we aliased the namespace in
hms_client.h to 'hive'. This doesn't really work well, I've seen issues
in tests where for unknown reasons the hive namespace doesn't work.

A better fix is to just edit hive_metastore.thrift to change the
generated namespace, since it's vendored anyway. This exposed that our
CMake thrift generator doesn't properly consider the input .thrift files
to be a dependency of the code generation step, so I had to fix that as
well to get this to compile without blowing away the build dir.

Change-Id: I1b4d823a0329c63c9ce830f855c3d1021fbc08cf
Reviewed-on: http://gerrit.cloudera.org:8080/9553
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b525378c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b525378c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b525378c

Branch: refs/heads/master
Commit: b525378c550a8227257ffac2a8d45a6d519f33f7
Parents: debcb8e
Author: Dan Burkert 
Authored: Wed Mar 7 19:03:21 2018 -0800
Committer: Dan Burkert 
Committed: Thu Mar 8 04:29:21 2018 +

--
 cmake_modules/FindThrift.cmake | 1 +
 src/kudu/hms/hive_metastore.thrift | 5 -
 src/kudu/hms/hms_client.h  | 2 --
 3 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/b525378c/cmake_modules/FindThrift.cmake
--
diff --git a/cmake_modules/FindThrift.cmake b/cmake_modules/FindThrift.cmake
index 4016c20..834bbfd 100644
--- a/cmake_modules/FindThrift.cmake
+++ b/cmake_modules/FindThrift.cmake
@@ -108,6 +108,7 @@ function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
 
 add_custom_command(
   OUTPUT ${THRIFT_CC_OUT} ${THRIFT_H_OUT}
+  DEPENDS ${ABS_FIL}
   COMMAND  ${THRIFT_EXECUTABLE}
   ARGS
 --gen cpp:moveable_types

http://git-wip-us.apache.org/repos/asf/kudu/blob/b525378c/src/kudu/hms/hive_metastore.thrift
--
diff --git a/src/kudu/hms/hive_metastore.thrift 
b/src/kudu/hms/hive_metastore.thrift
index 448ce6c..969f4f8 100644
--- a/src/kudu/hms/hive_metastore.thrift
+++ b/src/kudu/hms/hive_metastore.thrift
@@ -23,6 +23,9 @@
 # With backports:
 #   - HIVE-16993
 #
+# With edits:
+#   - Change cpp namespace to 'hive' to match the Kudu codebase style.
+#
 # Before updating to a new version, consider that Kudu must remain compatible
 # with a range of Hive Metastore versions.
 
@@ -34,7 +37,7 @@ include "share/fb303/if/fb303.thrift"
 
 namespace java org.apache.hadoop.hive.metastore.api
 namespace php metastore
-namespace cpp Apache.Hadoop.Hive
+namespace cpp hive
 
 const string DDL_TIME = "transient_lastDdlTime"
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/b525378c/src/kudu/hms/hms_client.h
--
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index a71fcf6..21f18f9 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -28,8 +28,6 @@
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 
-namespace hive = Apache::Hadoop::Hive;
-
 namespace kudu {
 
 class HostPort;

kudu git commit: Add CatalogManager::master_consensus() accessor

[danburkert]

Repository: kudu
Updated Branches:
  refs/heads/master 74dc5da86 -> 954d9f71b


Add CatalogManager::master_consensus() accessor

A patch I'm working on is going to start calling
Master::GetMasterHostPorts from inside of CatalogManager::Init.
GetMasterHostPorts calls into the catalog manager, and checks that it is
initialized. To break this circular dependency this introduces a new
accessor for the master tablet RaftConsensus instance which becomes
available immediately after the tablet is initialized. A few call-sites
are switched over to this accessor instead of drilling into catalog
manager.

Change-Id: Ie6887900329e67222da129b4a1b532cfb0a364b4
Reviewed-on: http://gerrit.cloudera.org:8080/9541
Reviewed-by: Adar Dembo 
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/954d9f71
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/954d9f71
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/954d9f71

Branch: refs/heads/master
Commit: 954d9f71b50e2fae24a6e3acfe094b4d24dc5c2c
Parents: 74dc5da
Author: Dan Burkert 
Authored: Wed Mar 7 15:06:11 2018 -0800
Committer: Dan Burkert 
Committed: Thu Mar 8 00:39:23 2018 +

--
 src/kudu/integration-tests/security-master-auth-itest.cc |  9 ++---
 src/kudu/master/catalog_manager.cc   | 11 +++
 src/kudu/master/catalog_manager.h|  5 +
 src/kudu/master/master.cc|  9 ++---
 4 files changed, 20 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/954d9f71/src/kudu/integration-tests/security-master-auth-itest.cc
--
diff --git a/src/kudu/integration-tests/security-master-auth-itest.cc 
b/src/kudu/integration-tests/security-master-auth-itest.cc
index c8eca38..9bdede1 100644
--- a/src/kudu/integration-tests/security-master-auth-itest.cc
+++ b/src/kudu/integration-tests/security-master-auth-itest.cc
@@ -22,16 +22,13 @@
 #include 
 
 #include "kudu/consensus/raft_consensus.h"
-#include "kudu/gutil/ref_counted.h"
 #include "kudu/master/catalog_manager.h"
 #include "kudu/master/master.h"
 #include "kudu/master/mini_master.h"
-#include "kudu/master/sys_catalog.h"
 #include "kudu/mini-cluster/internal_mini_cluster.h"
 #include "kudu/rpc/messenger.h"
 #include "kudu/security/tls_context.h"
 #include "kudu/security/token_verifier.h"
-#include "kudu/tablet/tablet_replica.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
@@ -84,8 +81,7 @@ TEST_F(SecurityMasterAuthTest, FollowerCertificates) {
 ASSERT_TRUE(tls.has_cert());
   }
 
-  auto* consensus = cluster_->mini_master(0)->master()->catalog_manager()->
-  sys_catalog()->tablet_replica()->consensus();
+  auto consensus = 
cluster_->mini_master(0)->master()->catalog_manager()->master_consensus();
   ASSERT_OK(consensus->StartElection(
   RaftConsensus::ELECT_EVEN_IF_LEADER_IS_ALIVE,
   RaftConsensus::EXTERNAL_REQUEST));
@@ -105,8 +101,7 @@ TEST_F(SecurityMasterAuthTest, FollowerCertificates) {
 // the rest have always been followers. This is a test to cover regressions of
 // KUDU-2319, if any.
 TEST_F(SecurityMasterAuthTest, FollowerTokenVerificationKeys) {
-  auto* consensus = cluster_->mini_master(0)->master()->catalog_manager()->
-  sys_catalog()->tablet_replica()->consensus();
+  auto consensus = 
cluster_->mini_master(0)->master()->catalog_manager()->master_consensus();
   ASSERT_OK(consensus->StartElection(
   RaftConsensus::ELECT_EVEN_IF_LEADER_IS_ALIVE,
   RaftConsensus::EXTERNAL_REQUEST));

http://git-wip-us.apache.org/repos/asf/kudu/blob/954d9f71/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 974927e..2aa09af 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -3755,6 +3755,17 @@ Status CatalogManager::ProcessTabletReport(
   return Status::OK();
 }
 
+std::shared_ptr CatalogManager::master_consensus() const {
+  // CatalogManager::InitSysCatalogAsync takes lock_ in exclusive mode in order
+  // to initialize sys_catalog_, so it's sufficient to take lock_ in shared 
mode
+  // here to protect access to sys_catalog_.
+  shared_lock l(lock_);
+  if (!sys_catalog_) {
+return nullptr;
+  }
+  return sys_catalog_->tablet_replica()->shared_consensus();
+}
+
 void CatalogManager::SendAlterTableRequest(const scoped_refptr& 
table) {
   vector tablets;
   table->GetAllTablets();