Repository: kudu
Updated Branches:
refs/heads/master 2d6f8ffa9 -> 8d6cfe10d
Bump Sentry and Hadoop versions
The motivation is to get access to SENTRY-2371, which introduces a new
Thrift interface that will be used by Kudu to retrieve user privileges.
SENTRY-2371 has not been released yet, so the new Sentry version is
based on the latest commit of the master branch. The new Sentry version
appears to have an incompatibility with Hadoop 2.8.2, so this also
includes a bump to the latest Hadoop 2.8.x version. I suspect the fix
between 2.8.2 and 2.8.5 was introduced in [1], but the commit message is
vague and I haven't dug any further. The incompatiblity manifests as an
exception during Sentry startup:
java.lang.NoSuchMethodError:
org.apache.hadoop.conf.Configuration.addResource(Ljava/net/URL;Z)V
at
org.apache.sentry.service.thrift.SentryService.loadConfig(SentryService.java:576)
at
org.apache.sentry.service.thrift.SentryService$CommandImpl.run(SentryService.java:600)
at org.apache.sentry.SentryMain.main(SentryMain.java:120)
[1]:
https://github.com/apache/hadoop/commit/7af9b8ad1e993ef791aa38740b6aabc4c233a30f
Change-Id: I8bcc4ff6fac0435b037b984f45da75bed6ff4be5
Reviewed-on: http://gerrit.cloudera.org:8080/11601
Reviewed-by: Hao Hao <hao....@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8d6cfe10
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8d6cfe10
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8d6cfe10
Branch: refs/heads/master
Commit: 8d6cfe10d47bdb422c6cc0e770afbd70d6ec7adf
Parents: 2d6f8ff
Author: Dan Burkert <danburk...@apache.org>
Authored: Fri Oct 5 14:10:32 2018 -0700
Committer: Dan Burkert <danburk...@apache.org>
Committed: Mon Oct 8 17:59:17 2018 +0000
----------------------------------------------------------------------
src/kudu/sentry/sentry_common_service.thrift | 3 +-
src/kudu/sentry/sentry_policy_service.thrift | 100 ++++++++++++++++++++--
thirdparty/vars.sh | 6 +-
3 files changed, 99 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/src/kudu/sentry/sentry_common_service.thrift
----------------------------------------------------------------------
diff --git a/src/kudu/sentry/sentry_common_service.thrift
b/src/kudu/sentry/sentry_common_service.thrift
index e37f4f6..b8a730e 100644
--- a/src/kudu/sentry/sentry_common_service.thrift
+++ b/src/kudu/sentry/sentry_common_service.thrift
@@ -19,7 +19,7 @@
*/
# DO NOT MODIFY! Copied from
-#
https://raw.githubusercontent.com/apache/sentry/release-2.0.1/sentry-provider/sentry-provider-db/src/main/resources/sentry_common_service.thrift
+#
https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_common_service.thrift
#
# With edits:
# - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -47,4 +47,3 @@ struct TSentryResponseStatus {
2: required string message
3: optional string stack
}
-
http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/src/kudu/sentry/sentry_policy_service.thrift
----------------------------------------------------------------------
diff --git a/src/kudu/sentry/sentry_policy_service.thrift
b/src/kudu/sentry/sentry_policy_service.thrift
index a11872b..d32e39b 100644
--- a/src/kudu/sentry/sentry_policy_service.thrift
+++ b/src/kudu/sentry/sentry_policy_service.thrift
@@ -19,7 +19,7 @@
*/
# DO NOT MODIFY! Copied from
-#
https://raw.githubusercontent.com/apache/sentry/release-2.0.1/sentry-provider/sentry-provider-db/src/main/resources/sentry_policy_service.thrift
+#
https://raw.githubusercontent.com/apache/sentry/2c9a927a9e87cba0e4c0f34fc0b55887c6636927/sentry-service/sentry-service-api/src/main/resources/sentry_policy_service.thrift
#
# With edits:
# - Change cpp namespace to 'sentry' to match the Kudu codebase style.
@@ -30,8 +30,8 @@
include "sentry_common_service.thrift"
-namespace java org.apache.sentry.provider.db.service.thrift
-namespace php sentry.provider.db.service.thrift
+namespace java org.apache.sentry.api.service.thrift
+namespace php sentry.api.service.thrift
namespace cpp sentry
enum TSentryGrantOption {
@@ -44,6 +44,12 @@ enum TSentryGrantOption {
UNSET = -1
}
+enum TSentryPrincipalType {
+ NONE = 0,
+ ROLE = 1,
+ USER = 2
+}
+
# Represents a Privilege in transport from the client to the server
struct TSentryPrivilege {
1: required string privilegeScope, # Valid values are SERVER, DATABASE, TABLE,
COLUMN, URI
@@ -190,9 +196,18 @@ struct TSentryAuthorizable {
struct TListSentryPrivilegesRequest {
1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
2: required string requestorUserName, # user on whose behalf the request is
issued
+
+# @Deprecated Use principalName instead to set role names or user names. This
parameter will be
+# removed in the next major version of Sentry 3.0
4: required string roleName, # get privileges assigned for this role
-5: optional TSentryAuthorizable authorizableHierarchy # get privileges
assigned for this role
+5: optional TSentryAuthorizable authorizableHierarchy, # get privileges
assigned for this role
+
+# Get privileges assigned for this principal name. This principalName should
be set to a role name
+# or user name depending of which function you call, either
list_sentry_privileges_by_role or
+# list_sentry_privileges_by_user
+6: optional string principalName
}
+
struct TListSentryPrivilegesResponse {
1: required sentry_common_service.TSentryResponseStatus status
2: optional set<TSentryPrivilege> privileges
@@ -250,11 +265,19 @@ struct TListSentryPrivilegesByAuthRequest {
2: required string requestorUserName, # user on whose behalf the request is
issued
3: required set<TSentryAuthorizable> authorizableSet,
4: optional set<string> groups,
-5: optional TSentryActiveRoleSet roleSet
+5: optional TSentryActiveRoleSet roleSet,
+6: optional set<string> users
}
struct TListSentryPrivilegesByAuthResponse {
1: required sentry_common_service.TSentryResponseStatus status,
-2: optional map<TSentryAuthorizable, TSentryPrivilegeMap> privilegesMapByAuth
# will not be set in case of an error
+
+# privilegesMapByAuth (legacy & compatible parameter) contains role privileges
+# (will not be set in case of an error)
+2: optional map<TSentryAuthorizable, TSentryPrivilegeMap> privilegesMapByAuth,
+
+# privilegesMapByAuthForUsers contains user privileges
+# (will not be set in case of an error)
+3: optional map<TSentryAuthorizable, TSentryPrivilegeMap>
privilegesMapByAuthForUsers
}
# Obtain a config value from the Sentry service
@@ -329,6 +352,50 @@ struct TSentrySyncIDResponse {
2: required i64 id // Most recent processed ID
}
+/*
+ * This request is an extension to TSentrySyncIDRequest. Additionally this
request
+ * is used to update the HMS events and the owner changes associated with
events.
+ * To be backward compatible, TSentrySyncIDRequest is not updated. Instead new
request
+ * is created extending it.
+*/
+
+struct TSentryHmsEventNotification {
+1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
+2: required string requestorUserName, # user on whose behalf the request is
issued
+3: required i64 id, # Requested ID
+# Constructed from enum
org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType
+4: required string eventType, # Type of the event which resulted in owner
update request
+5: required TSentryAuthorizable authorizable, # Authorizable object
+6: optional TSentryPrincipalType ownerType, # Type of the owner
+7: optional string ownerName # owner name
+
+}
+
+struct TSentryHmsEventNotificationResponse {
+1: required sentry_common_service.TSentryResponseStatus status
+2: required i64 id // Most recent processed ID
+}
+
+/**
+* API that requests all roles and users privileges from the Sentry server.
+**/
+struct TSentryPrivilegesRequest {
+1: required i32 protocol_version = sentry_common_service.TSENTRY_SERVICE_V2,
+2: required string requestorUserName # user on whose behalf the request is
issued
+}
+
+/**
+* API that returns either all users or roles privileges found on the Sentry
server.
+*
+* The response returns a mapping object that maps the role or user name to the
privileges
+* they have in the server. An empty set of privileges may be returned to each
role or user
+* name. Null values are not returned.
+**/
+struct TSentryPrivilegesResponse {
+1: required sentry_common_service.TSentryResponseStatus status
+2: required map<string, set<TSentryPrivilege>> privilegesMap;
+}
+
service SentryPolicyService
{
TCreateSentryRoleResponse create_sentry_role(1:TCreateSentryRoleRequest
request)
@@ -346,7 +413,16 @@ service SentryPolicyService
TListSentryRolesResponse
list_sentry_roles_by_group(1:TListSentryRolesRequest request)
TListSentryRolesResponse
list_sentry_roles_by_user(1:TListSentryRolesForUserRequest request)
+ # List sentry privileges granted to the given role, filterted
+ # based on authorization hierarchy if present.
TListSentryPrivilegesResponse
list_sentry_privileges_by_role(1:TListSentryPrivilegesRequest request)
+ # List sentry privileges granted to the given user, filterted
+ # based on authorization hierarchy if present.
+ TListSentryPrivilegesResponse
list_sentry_privileges_by_user(1:TListSentryPrivilegesRequest request)
+ # List sentry privileges granted to the given user and the groups
+ # the user associated with, filterted based on authorization
+ # hierarchy if present.
+ TListSentryPrivilegesResponse
list_sentry_privileges_by_user_and_itsgroups(1:TListSentryPrivilegesRequest
request)
# For use with ProviderBackend.getPrivileges only
TListSentryPrivilegesForProviderResponse
list_sentry_privileges_for_provider(1:TListSentryPrivilegesForProviderRequest
request)
@@ -367,4 +443,16 @@ service SentryPolicyService
# Synchronize between HMS notifications and Sentry
TSentrySyncIDResponse sentry_sync_notifications(1:TSentrySyncIDRequest
request);
+
+ # Notify Sentry about new events in HMS. Currently used to synchronize
between HMS/Sentry
+ # and also update sentry with the owner information.
+ TSentryHmsEventNotificationResponse
sentry_notify_hms_event(1:TSentryHmsEventNotification request);
+
+ # Returns a map of all roles and their privileges that exist in the Sentry
server.
+ # The mapping object returned will be in the form of [roleName,
set<privileges>]
+ TSentryPrivilegesResponse list_roles_privileges(1:TSentryPrivilegesRequest
request);
+
+ # Returns a map of all users and their privileges that exist in the Sentry
server.
+ # The mapping object returned will be in the form of [userName,
set<privileges>]
+ TSentryPrivilegesResponse list_users_privileges(1:TSentryPrivilegesRequest
request);
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/8d6cfe10/thirdparty/vars.sh
----------------------------------------------------------------------
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index fe25293..fe4cd10 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -222,10 +222,12 @@ HIVE_SOURCE=$TP_SOURCE_DIR/$HIVE_NAME
# Note: The Hadoop release tarball is stripped of unnecessary jars before being
# uploaded. See thirdparty/package-hadoop.sh for details.
-HADOOP_VERSION=2.8.2
+HADOOP_VERSION=2.8.5
HADOOP_NAME=hadoop-$HADOOP_VERSION
HADOOP_SOURCE=$TP_SOURCE_DIR/$HADOOP_NAME
-SENTRY_VERSION=2.0.1
+# TODO(dan): bump to a release version once SENTRY-2371 is published. The SHA
+# below is the current head of the master branch.
+SENTRY_VERSION=2c9a927a9e87cba0e4c0f34fc0b55887c6636927
SENTRY_NAME=apache-sentry-$SENTRY_VERSION-bin
SENTRY_SOURCE=$TP_SOURCE_DIR/$SENTRY_NAME
