[hms] enable 'notifications.add.thrift.objects' in mini HMS This commits explictly enables 'notifications.add.thrift.objects' to add the entire thrift Table/Partition objects to Hive notification log. So that Kudu can get all the required fields to properly parse the notification messages, when running with Hive distributions that disable it by default.
Change-Id: I9f28f52bdddd59c5b0dc7120d62482de8becb1ee Reviewed-on: http://gerrit.cloudera.org:8080/11566 Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ce2b5277 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ce2b5277 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ce2b5277 Branch: refs/heads/master Commit: ce2b52770be4a81800fb5eb155b952d5211e2017 Parents: 63e4a0a Author: Hao Hao <[email protected]> Authored: Mon Oct 1 13:05:27 2018 -0700 Committer: Hao Hao <[email protected]> Committed: Wed Oct 3 18:24:32 2018 +0000 ---------------------------------------------------------------------- src/kudu/hms/hms_client.cc | 22 ++++++++++++++++++++-- src/kudu/hms/hms_client.h | 1 + src/kudu/hms/mini_hms.cc | 9 +++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/ce2b5277/src/kudu/hms/hms_client.cc ---------------------------------------------------------------------- diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc index ef37616..9d51956 100644 --- a/src/kudu/hms/hms_client.cc +++ b/src/kudu/hms/hms_client.cc @@ -112,6 +112,8 @@ const char* const HmsClient::kStorageHandlerKey = "storage_handler"; const char* const HmsClient::kKuduMetastorePlugin = "org.apache.kudu.hive.metastore.KuduMetastorePlugin"; const char* const HmsClient::kHiveFilterFieldParams = "hive_filter_field_params__"; +const char* const HmsClient::kNotificationAddThriftObjects = + "hive.metastore.notifications.add.thrift.objects"; const char* const HmsClient::kManagedTable = "MANAGED_TABLE"; const char* const HmsClient::kExternalTable = "EXTERNAL_TABLE"; @@ -174,10 +176,26 @@ Status HmsClient::Start() { if (boost::iequals(disallow_incompatible_column_type_changes, "true")) { return Status::IllegalState(Substitute( - "Hive Metastore configuration is invalid: $0 must be set to false", - kDisallowIncompatibleColTypeChanges)); + "Hive Metastore configuration is invalid: $0 must be set to false", + kDisallowIncompatibleColTypeChanges)); } + // Check that the HMS is configured to add the entire thrift Table/Partition + // objects to the HMS notifications, which is required to properly parse the + // HMS notification log. This is specific to the HMS version shipped in + // Cloudera's CDH. + string thrift_objects_config; + HMS_RET_NOT_OK(client_.get_config_value(thrift_objects_config, + kNotificationAddThriftObjects, + "true"), + Substitute("failed to get Hive Metastore $0 configuration", + kNotificationAddThriftObjects)); + if (boost::iequals(thrift_objects_config, "false")) { + return Status::IllegalState(Substitute( + "Hive Metastore configuration is invalid: $0 must be set to true", + kNotificationAddThriftObjects)); + } + return Status::OK(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/ce2b5277/src/kudu/hms/hms_client.h ---------------------------------------------------------------------- diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h index c081366..23451f3 100644 --- a/src/kudu/hms/hms_client.h +++ b/src/kudu/hms/hms_client.h @@ -83,6 +83,7 @@ class HmsClient { static const char* const kDisallowIncompatibleColTypeChanges; static const char* const kDbNotificationListener; static const char* const kKuduMetastorePlugin; + static const char* const kNotificationAddThriftObjects; // See org.apache.hadoop.hive.metastore.TableType. static const char* const kManagedTable; http://git-wip-us.apache.org/repos/asf/kudu/blob/ce2b5277/src/kudu/hms/mini_hms.cc ---------------------------------------------------------------------- diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc index 253de09..079cdb1 100644 --- a/src/kudu/hms/mini_hms.cc +++ b/src/kudu/hms/mini_hms.cc @@ -207,6 +207,10 @@ Status MiniHms::CreateHiveSite() const { // - hive.support.special.characters.tablename // Configures the HMS to allow special characters such as '/' in table // names. + // + // - hive.metastore.notifications.add.thrift.objects + // Configured the HMS to add the entire thrift Table/Partition + // objects to the HMS notifications. static const string kFileTemplate = R"( <configuration> <property> @@ -271,6 +275,11 @@ Status MiniHms::CreateHiveSite() const { <name>hive.support.special.characters.tablename</name> <value>true</value> </property> + + <property> + <name>hive.metastore.notifications.add.thrift.objects</name> + <value>true</value> + </property> </configuration> )";
