Repository: sentry Updated Branches: refs/heads/master bee44c28a -> 86c12d64e
SENTRY-2275: Grant and revoke owner privileges based on HMS updates(client-side). (Kalyan Kumar kalvagadda, reviewed-by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/86c12d64 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/86c12d64 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/86c12d64 Branch: refs/heads/master Commit: 86c12d64ee47d2644a7bcda7277bedcae98c8aba Parents: bee44c2 Author: Kalyan Kumar Kalvagadda <[email protected]> Authored: Wed Jun 20 07:22:36 2018 -0500 Committer: Kalyan Kumar Kalvagadda <[email protected]> Committed: Wed Jun 20 07:22:36 2018 -0500 ---------------------------------------------------------------------- .../binding/metastore/SentryHmsEvent.java | 221 ++++++++++++ ...rySyncHMSNotificationsPostEventListener.java | 174 ++++++---- ...rySyncHMSNotificationsPostEventListener.java | 337 +++++++++++++++++-- .../thrift/SentryPolicyServiceClient.java | 9 + .../SentryPolicyServiceClientDefaultImpl.java | 11 + 5 files changed, 662 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java new file mode 100644 index 0000000..42be3c3 --- /dev/null +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHmsEvent.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.binding.metastore; + +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.CreateTableEvent; +import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; +import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; +import org.apache.sentry.api.service.thrift.TSentryAuthorizable; +import org.apache.sentry.api.service.thrift.TSentryHmsEventNotification; +import org.apache.sentry.api.service.thrift.TSentryObjectOwnerType; + +import java.util.Map; + +/** + * Sentry HMS Event holds the information from the HMS Event that sentry is interested in + */ +class SentryHmsEvent { + // eventId is used to synchronize the event. If synchronization is not needed it is initialized to 0. + private long eventId; + private final EventType eventType; + private String ownerName; + private TSentryObjectOwnerType ownerType; + private TSentryAuthorizable authorizable; + private final Boolean isMetastoreTransactionActive; + + private static final Map<PrincipalType, TSentryObjectOwnerType> mapOwnerType = ImmutableMap.of( + PrincipalType.ROLE, TSentryObjectOwnerType.ROLE, + PrincipalType.USER, TSentryObjectOwnerType.USER + ); + + /** + * Construct SentryHmsEvent from ListenerEvent and event Type + * + * event and transaction information is initialized. + * @param event ListenerEvent + * @param type EventType + */ + private SentryHmsEvent(ListenerEvent event, EventType type) { + String isActive = event.getParameters().getOrDefault( + MetaStoreEventListenerConstants.HIVE_METASTORE_TRANSACTION_ACTIVE, null); + isMetastoreTransactionActive = (isActive != null) && Boolean.valueOf(isActive); + eventId = getEventId(event); + eventType = type; + } + + /** + * Construct SentryHmsEvent from CreateTableEvent + * + * event, transaction, owner and authorizable info is initialized from event. + * @param event CreateTableEvent + */ + public SentryHmsEvent(CreateTableEvent event) { + this(event, EventType.CREATE_TABLE); + setOwnerInfo(event.getTable()); + setAuthorizable(event.getTable()); + } + + /** + * Construct SentryHmsEvent from DropTableEvent + * + * event, transaction, owner and authorizable info is initialized from event. + * @param event DropTableEvent + */ + public SentryHmsEvent(DropTableEvent event) { + this(event, EventType.DROP_TABLE); + setOwnerInfo(event.getTable()); + setAuthorizable(event.getTable()); + } + + /** + * Construct SentryHmsEvent from AlterTableEvent + * + * event, transaction, owner and authorizable info is initialized from event. + * @param event AlterTableEvent + */ + public SentryHmsEvent(AlterTableEvent event) { + this(event, EventType.ALTER_TABLE); + if(!StringUtils.equals(event.getOldTable().getOwner(), event.getNewTable().getOwner())) { + // Owner Changed. + setOwnerInfo(event.getNewTable()); + } + setAuthorizable(event.getNewTable()); + } + + /** + * Construct SentryHmsEvent from CreateDatabaseEvent + * + * event, transaction, owner and authorizable info is initialized from event. + * @param event CreateDatabaseEvent + */ + public SentryHmsEvent(CreateDatabaseEvent event) { + this(event, EventType.CREATE_DATABASE); + setOwnerInfo(event.getDatabase()); + setAuthorizable(event.getDatabase()); + } + + /** + * Construct SentryHmsEvent from DropDatabaseEvent + * + * event, transaction, owner and authorizable info is initialized from event. + * @param event DropDatabaseEvent + */ + public SentryHmsEvent(DropDatabaseEvent event) { + this(event, EventType.DROP_DATABASE); + setOwnerInfo(event.getDatabase()); + setAuthorizable(event.getDatabase()); + } + + public EventType getEventType() { + return eventType; + } + + public long getEventId() { + return eventId; + } + + private void setOwnerInfo(Table table) { + ownerName = (table != null) ? table.getOwner() : null; + // Hive 2.3.2 currently support owner type. Assuming user as the type for now. + // TODO once sentry dependency is changed to a hive version that suppots user type for table this + // hard coding should be rempved. + ownerType = TSentryObjectOwnerType.USER; + } + + private void setOwnerInfo(Database database) { + ownerName = (database != null) ? database.getOwnerName() : null; + ownerType = (database != null) ? + getTSentryHmsObjectOwnerType(database.getOwnerType()) : null; + } + + private void setAuthorizable(Table table) { + if (authorizable == null) { + authorizable = new TSentryAuthorizable(); + } + authorizable.setDb((table != null) ? table.getDbName() : null); + authorizable.setTable((table != null) ? table.getTableName() : null); + } + + private void setAuthorizable(Database database) { + if (authorizable == null) { + authorizable = new TSentryAuthorizable(); + } + authorizable.setDb((database != null) ? database.getName() : null); + } + + /** + * Updates the event_id + * @param eventId event id + */ + public void setEventId(long eventId) { + this.eventId = eventId; + } + + /** + * Constructs notification message that is sent to sentry server. + * + * @return notification event. + */ + public TSentryHmsEventNotification getHmsEventNotification() { + TSentryHmsEventNotification updateAndSyncIDRequest = new TSentryHmsEventNotification(); + updateAndSyncIDRequest.setOwnerName(ownerName); + updateAndSyncIDRequest.setOwnerType(ownerType); + updateAndSyncIDRequest.setAuthorizable(authorizable); + updateAndSyncIDRequest.setId(eventId); + updateAndSyncIDRequest.setEventType(eventType.toString()); + return updateAndSyncIDRequest; + } + + /** + * Converts Principle to Owner Type defined by sentry. + * + * @param principalType Hive Principle Type + * @return TSentryObjectOwnerType if the input is valid else null + */ + private TSentryObjectOwnerType getTSentryHmsObjectOwnerType(PrincipalType principalType) { + return mapOwnerType.get(principalType); + } + + /** + * Gets event-id from Event + * + * @param event HMS Event + * @return returns the eventId extracted from Event OR -1 if the eventId is not found the event provided. + */ + private long getEventId(ListenerEvent event) { + return Long.parseLong(event.getParameters().getOrDefault( + MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, "-1")); + } + + /** + * @return True if the HMS is calling this notification in an active transaction; False otherwise + */ + public Boolean isMetastoreTransactionActive() { + return isMetastoreTransactionActive; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java index ccb60ff..f7d1b07 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentrySyncHMSNotificationsPostEventListener.java @@ -18,11 +18,14 @@ package org.apache.sentry.binding.metastore; import com.google.common.annotations.VisibleForTesting; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.MetaStoreEventListener; -import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; @@ -33,15 +36,13 @@ import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient; import org.apache.sentry.service.thrift.SentryServiceClientFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - /** * This HMS post-event listener is used only to synchronize with HMS notifications on the Sentry server * whenever a DDL event happens on the Hive metastore. @@ -84,19 +85,48 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL authzConf = HiveAuthzConf.getAuthzConf((HiveConf) config); } + /** + * Notify sentry server when new table is created + * + * @param tableEvent Create table Event + * @throws MetaException + */ @Override public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { - syncNotificationEvents(tableEvent, "onCreateTable"); + // Failure event, Need not be notified. + if (failedEvent(tableEvent, EventType.CREATE_TABLE)) { + return; + } + SentryHmsEvent event = new SentryHmsEvent(tableEvent); + notifyHmsEvent(event); } + /** + * Notify sentry server when table is dropped + * + * @param tableEvent Drop table event + * @throws MetaException + */ @Override public void onDropTable(DropTableEvent tableEvent) throws MetaException { - syncNotificationEvents(tableEvent, "onDropTable"); + // Failure event, Need not be notified. + if (failedEvent(tableEvent, EventType.DROP_TABLE)) { + return; + } + SentryHmsEvent event = new SentryHmsEvent(tableEvent); + notifyHmsEvent(event); } + /** + * Notify sentry server when when table is altered. + * Owner information is updated in the request only when there is owner change. + * Sentry is not notified when neither rename happened nor owner is changed + * + * @param tableEvent Alter table event + * @throws MetaException When both the owner change and rename is seen. + */ @Override public void onAlterTable(AlterTableEvent tableEvent) throws MetaException { - if (tableEvent == null) { return; } @@ -111,16 +141,20 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL if (newTable == null) { return; } + // Failure event, Need not be notified. + if (failedEvent(tableEvent, EventType.ALTER_TABLE)) { + return; + } - String oldDbName = oldTable.getDbName(); - String newDbName = newTable.getDbName(); - String oldTableName = oldTable.getTableName(); - String newTableName = newTable.getTableName(); - - if (!newDbName.equalsIgnoreCase(oldDbName) || !newTableName.equalsIgnoreCase(oldTableName)) { - // make sure sentry gets the table rename event - syncNotificationEvents(tableEvent, "onAlterTable"); + if(StringUtils.equals(oldTable.getOwner(), newTable.getOwner()) && + StringUtils.equalsIgnoreCase(oldTable.getDbName(), newTable.getDbName()) && + StringUtils.equalsIgnoreCase(oldTable.getTableName(), newTable.getTableName())) { + // nothing to notify, neither rename happened nor owner is changed + return; } + + SentryHmsEvent event = new SentryHmsEvent(tableEvent); + notifyHmsEvent(event); } @Override @@ -138,76 +172,73 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL // no-op } + /** + * Notify sentry server when new database is created + * + * @param dbEvent Create database event + * @throws MetaException + */ @Override public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException { - syncNotificationEvents(dbEvent, "onCreateDatabase"); + // Failure event, Need not be notified. + if (failedEvent(dbEvent, EventType.CREATE_DATABASE)) { + return; + } + SentryHmsEvent event = new SentryHmsEvent(dbEvent); + notifyHmsEvent(event); } + /** + * Notify sentry server when database is dropped + * + * @param dbEvent Drop database event. + * @throws MetaException + */ @Override public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { - syncNotificationEvents(dbEvent, "onDropDatabase"); + // Failure event, Need not be notified. + if (failedEvent(dbEvent, EventType.DROP_DATABASE)) { + return; + } + SentryHmsEvent event = new SentryHmsEvent(dbEvent); + notifyHmsEvent(event); } /** - * It requests the Sentry server the synchronization of recent notification events. + * Notifies sentry server about the HMS Event and related metadata. * - * After the sync call, the latest processed ID will be stored for future reference to avoid - * syncing an ID that was already processed. - * - * @param event An event that contains a DB_NOTIFICATION_EVENT_ID_KEY_NAME value to request. + * @param event Sentry HMS event. */ - private void syncNotificationEvents(ListenerEvent event, String eventName) { - // Do not sync notifications if the event has failed. - if (failedEvent(event, eventName)) { - return; - } - - Map<String, String> eventParameters = event.getParameters(); - if (!eventParameters.containsKey(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) { - return; - } - + private void notifyHmsEvent(SentryHmsEvent event ) { /* If the HMS is running in an active transaction, then we do not want to sync with Sentry * because the desired eventId is not available for Sentry yet, and Sentry may block the HMS - * forever or until a read time-out happens. */ - if (isMetastoreTransactionActive(eventParameters)) { + * forever or until a read time-out happens. + * */ + if(event.isMetastoreTransactionActive()) { return; } - long eventId = - Long.parseLong(eventParameters.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)); - - // This check is only for performance reasons to avoid calling the sync thrift call if the Sentry server - // already processed the requested eventId. - if (eventId <= latestProcessedId.get()) { - return; + if (!shouldSyncEvent(event)) { + event.setEventId(0L); } - try(SentryPolicyServiceClient sentryClient = this.getSentryServiceClient()) { - LOGGER.debug("Starting Sentry/HMS notifications sync for {} (id: {})", eventName, eventId); - long sentryLatestProcessedId = sentryClient.syncNotifications(eventId); - LOGGER.debug("Finished Sentry/HMS notifications sync for {} (id: {})", eventName, eventId); + try (SentryPolicyServiceClient sentryClient = this.getSentryServiceClient()) { + LOGGER.debug("Notifying sentry about Notification for {} (id: {})", event.getEventType(), + event.getEventId()); + long sentryLatestProcessedId = sentryClient.notifyHmsNotification(event.getHmsEventNotification()); + LOGGER.debug("Finished Notifying sentry about Notification for {} (id: {})", event.getEventType(), + event.getEventId()); LOGGER.debug("Latest processed event ID returned by the Sentry server: {}", sentryLatestProcessedId); - updateProcessedId(sentryLatestProcessedId); } catch (Exception e) { // This error is only logged. There is no need to throw an error to Hive because HMS sync is called // after the notification is already generated by Hive (as post-event). - LOGGER.error("Failed to sync requested HMS notifications up to the event ID: " + eventId, e); + LOGGER.error("Encountered failure while notifying notification for {} (id: {})", + event.getEventType(), event.getEventId(), e); } } /** - * @return True if the HMS is calling this notification in an active transaction; False otherwise - */ - private boolean isMetastoreTransactionActive(Map<String, String> parameters) { - String transactionActive = - parameters.get(MetaStoreEventListenerConstants.HIVE_METASTORE_TRANSACTION_ACTIVE); - - return transactionActive != null && Boolean.valueOf(transactionActive); - } - - /** * Updates the latest processed ID, if and only if eventId is bigger. This keeps the contract that * {@link #latestProcessedId} may only increase. * @@ -223,7 +254,7 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL /** * Sets the sentry client object (for testing purposes only) - * + * <p> * It may be set by unit-tests as a mock object and used to verify that the client methods * were called correctly (see TestSentrySyncHMSNotificationsPostEventListener). */ @@ -245,13 +276,34 @@ public class SentrySyncHMSNotificationsPostEventListener extends MetaStoreEventL } } - private boolean failedEvent(ListenerEvent event, String eventName) { + private boolean failedEvent(ListenerEvent event, EventType eventType) { if (!event.getStatus()) { LOGGER.debug("Skip HMS synchronization request with the Sentry server for {} " + - "{} since the operation failed. \n", eventName, event); + "{} since the operation failed. \n", eventType.toString(), event); return true; } return false; } + + /** + * Performs checks to make sure if the event should be synced. + * + * @param event SentryHmsEvent + * @return False: if Event should not be synced, True otherwise. + */ + private boolean shouldSyncEvent(SentryHmsEvent event) { + + // Sync need not be performed, Event id is not updated in the event. + if(event.getEventId() < 0) { + return false; + } + + // This check is only for performance reasons to avoid calling the sync thrift call if the Sentry + // server already processed the requested eventId. + if (event.getEventId() <= latestProcessedId.get()) { + return false; + } + return true; + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java index fc1c3d5..8e79cac 100644 --- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java +++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/metastore/TestSentrySyncHMSNotificationsPostEventListener.java @@ -20,11 +20,20 @@ package org.apache.sentry.binding.metastore; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.messaging.EventMessage; +import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.sentry.api.service.thrift.TSentryAuthorizable; +import org.apache.sentry.api.service.thrift.TSentryHmsEventNotification; +import org.apache.sentry.api.service.thrift.TSentryObjectOwnerType; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.core.common.exception.SentryUserException; import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient; @@ -47,6 +56,12 @@ public class TestSentrySyncHMSNotificationsPostEventListener { private static final boolean SUCCESSFUL_STATUS = true; private static final boolean EVENT_ID_SET = true; private static final boolean EVENT_ID_UNSET = false; + private static final String DBNAME = "db1"; + private static final String TABLENAME = "table1"; + private static final String TABLENAME_NEW = "table_new"; + private static final String OWNER = "owner1"; + private static final String OWNER_NEW = "owner_new"; + private SentrySyncHMSNotificationsPostEventListener eventListener; private SentryPolicyServiceClient mockSentryClient; @@ -83,23 +98,45 @@ public class TestSentrySyncHMSNotificationsPostEventListener { } @Test - public void testEventsWithoutAnEventIdDoNotSyncNotifications() throws MetaException { - callAllEventsThatSynchronize(SUCCESSFUL_STATUS, EVENT_ID_UNSET); - Mockito.verifyZeroInteractions(mockSentryClient); - } - - @Test public void testSuccessfulEventsWithAnEventIdSyncNotifications() throws Exception { - long latestEventId = callAllEventsThatSynchronize(SUCCESSFUL_STATUS, EVENT_ID_SET); + int eventId = 1; + + callAllEventsThatSynchronize(EventType.CREATE_DATABASE, SUCCESSFUL_STATUS, eventId++); + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId - 1); + notification.setEventType(EventType.CREATE_DATABASE.toString()); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + + callAllEventsThatSynchronize(EventType.DROP_DATABASE, SUCCESSFUL_STATUS, eventId++); + notification.setId(eventId - 1); + notification.setEventType(EventType.DROP_DATABASE.toString()); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + + callAllEventsThatSynchronize(EventType.CREATE_TABLE, SUCCESSFUL_STATUS, eventId++); + notification.setId(eventId - 1); + notification.setEventType(EventType.CREATE_TABLE.toString()); + notification.setOwnerType(TSentryObjectOwnerType.USER); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + + long latestEventId = callAllEventsThatSynchronize(EventType.DROP_TABLE, SUCCESSFUL_STATUS, eventId++); + notification.setId(eventId - 1); + notification.setEventType(EventType.DROP_TABLE.toString()); + notification.setOwnerType(TSentryObjectOwnerType.USER); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); - for (int i=1; i<=latestEventId; i++) { - Mockito.verify( - mockSentryClient, Mockito.times(1) - ).syncNotifications(i); - } Mockito.verify( - mockSentryClient, Mockito.times((int)latestEventId) + mockSentryClient, Mockito.times((int) latestEventId) ).close(); Mockito.verifyNoMoreInteractions(mockSentryClient); @@ -110,52 +147,294 @@ public class TestSentrySyncHMSNotificationsPostEventListener { Mockito.doAnswer(new Answer<Long>() { @Override public Long answer(InvocationOnMock invocation) throws Throwable { - Long id = (Long)invocation.getArguments()[0]; - return id + 1; + TSentryHmsEventNotification notification = (TSentryHmsEventNotification) invocation.getArguments()[0]; + return notification.getId() + 1; } - }).when(mockSentryClient).syncNotifications(Mockito.anyLong()); + }).when(mockSentryClient).notifyHmsNotification(Mockito.anyObject()); long latestEventId = callAllEventsThatSynchronize(SUCCESSFUL_STATUS, EVENT_ID_SET); - - for (int i=1; i<=latestEventId; i+=2) { - Mockito.verify( - mockSentryClient, Mockito.times(1) - ).syncNotifications(i); - } + verifyInvocations(); Mockito.verify( - mockSentryClient, Mockito.times((int)latestEventId / 2) + mockSentryClient, Mockito.times((int) latestEventId) ).close(); + } - Mockito.verifyNoMoreInteractions(mockSentryClient); + @Test + public void notificationOnTableCreate() throws Exception { + int eventId = 1; + Table tb = new Table(); + tb.setDbName(DBNAME); + tb.setTableName(TABLENAME); + tb.setOwner(OWNER); + CreateTableEvent createTableEvent = new CreateTableEvent(tb, true, null); + setEventId(EVENT_ID_SET, createTableEvent, eventId); + eventListener.onCreateTable(createTableEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventMessage.EventType.CREATE_TABLE.toString()); + notification.setOwnerName(OWNER); + notification.setOwnerType(TSentryObjectOwnerType.USER); + notification.getAuthorizable().setDb(DBNAME); + notification.getAuthorizable().setTable(TABLENAME); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + + @Test + public void notificationOnTableDrop() throws Exception { + int eventId = 1; + Table tb = new Table(); + tb.setDbName(DBNAME); + tb.setTableName(TABLENAME); + tb.setOwner(OWNER); + DropTableEvent dropTableEvent = new DropTableEvent(tb, true, true, null); + setEventId(EVENT_ID_SET, dropTableEvent, eventId); + eventListener.onDropTable(dropTableEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventMessage.EventType.DROP_TABLE.toString()); + notification.setOwnerName(OWNER); + notification.setOwnerType(TSentryObjectOwnerType.USER); + notification.getAuthorizable().setDb(DBNAME); + notification.getAuthorizable().setTable(TABLENAME); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + + @Test + public void notificationOnDatabaseCreate() throws Exception { + int eventId = 1; + Database db = new Database(); + db.setName(DBNAME); + db.setOwnerName(OWNER); + db.setOwnerType(PrincipalType.USER); + CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(db, true, null); + setEventId(EVENT_ID_SET, createDatabaseEvent, eventId); + eventListener.onCreateDatabase(createDatabaseEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventMessage.EventType.CREATE_DATABASE.toString()); + notification.setOwnerName(OWNER); + notification.setOwnerType(TSentryObjectOwnerType.USER); + notification.getAuthorizable().setDb(DBNAME); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + + @Test + public void notificationOnDatabaseDrop() throws Exception { + int eventId = 1; + Database db = new Database(); + db.setName(DBNAME); + db.setOwnerName(OWNER); + db.setOwnerType(PrincipalType.USER); + DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(db, true, null); + setEventId(EVENT_ID_SET, dropDatabaseEvent, eventId); + eventListener.onDropDatabase(dropDatabaseEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventType.DROP_DATABASE.toString()); + notification.setOwnerName(OWNER); + notification.setOwnerType(TSentryObjectOwnerType.USER); + notification.getAuthorizable().setDb(DBNAME); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + + + @Test + public void notificationOnAlterTableOwnerChange() throws Exception { + int eventId = 1; + Table old_tb = new Table(); + old_tb.setDbName(DBNAME); + old_tb.setTableName(TABLENAME); + old_tb.setOwner(OWNER); + + Table new_tb = new Table(); + new_tb.setDbName(DBNAME); + new_tb.setTableName(TABLENAME); + new_tb.setOwner(OWNER_NEW); + + AlterTableEvent alterTableEvent = new AlterTableEvent(old_tb, new_tb, true, null); + setEventId(EVENT_ID_SET, alterTableEvent, eventId); + eventListener.onAlterTable(alterTableEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventType.ALTER_TABLE.toString()); + notification.setOwnerName(OWNER_NEW); + notification.setOwnerType(TSentryObjectOwnerType.USER); + notification.getAuthorizable().setDb(DBNAME); + notification.getAuthorizable().setTable(TABLENAME); + + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + + @Test + public void notificationOnAlterTableRename() throws Exception { + int eventId = 1; + Table old_tb = new Table(); + old_tb.setDbName(DBNAME); + old_tb.setTableName(TABLENAME); + old_tb.setOwner(OWNER); + + Table new_tb = new Table(); + new_tb.setDbName(DBNAME); + new_tb.setTableName(TABLENAME_NEW); + new_tb.setOwner(OWNER); + + AlterTableEvent alterTableEvent = new AlterTableEvent(old_tb, new_tb, true, null); + setEventId(EVENT_ID_SET, alterTableEvent, eventId); + eventListener.onAlterTable(alterTableEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventType.ALTER_TABLE.toString()); + notification.getAuthorizable().setDb(DBNAME); + notification.getAuthorizable().setTable(TABLENAME_NEW); + + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + + @Test + public void notificationOnAlterTableNoRenameAndOwnerChange() throws Exception { + int eventId = 1; + Table old_tb = new Table(); + old_tb.setDbName(DBNAME); + old_tb.setTableName(TABLENAME); + old_tb.setOwner(OWNER); + + Table new_tb = new Table(); + new_tb.setDbName(DBNAME); + new_tb.setTableName(TABLENAME); + new_tb.setOwner(OWNER); + + AlterTableEvent alterTableEvent = new AlterTableEvent(old_tb, new_tb, true, null); + setEventId(EVENT_ID_SET, alterTableEvent, eventId); + eventListener.onAlterTable(alterTableEvent); + + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(eventId); + notification.setEventType(EventType.ALTER_TABLE.toString()); + notification.getAuthorizable().setDb(DBNAME); + notification.getAuthorizable().setTable(TABLENAME); + + Mockito.verify( + mockSentryClient, Mockito.times(0) + ).notifyHmsNotification(Mockito.anyObject()); } private long callAllEventsThatSynchronize(boolean status, boolean eventIdSet) throws MetaException { long eventId = 0; - CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(null, status , null); + CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(null, status, null); setEventId(eventIdSet, createDatabaseEvent, ++eventId); eventListener.onCreateDatabase(createDatabaseEvent); - DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(null, status , null); + DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(null, status, null); setEventId(eventIdSet, dropDatabaseEvent, ++eventId); eventListener.onDropDatabase(dropDatabaseEvent); - CreateTableEvent createTableEvent = new CreateTableEvent(null, status , null); + CreateTableEvent createTableEvent = new CreateTableEvent(null, status, null); setEventId(eventIdSet, createTableEvent, ++eventId); eventListener.onCreateTable(createTableEvent); - DropTableEvent dropTableEvent = new DropTableEvent(null, status , false, null); + DropTableEvent dropTableEvent = new DropTableEvent(null, status, false, null); setEventId(eventIdSet, dropTableEvent, ++eventId); eventListener.onDropTable(dropTableEvent); return eventId; } + private long callAllEventsThatSynchronize(EventType event, boolean status, long eventId) throws MetaException { + switch (event) { + case CREATE_DATABASE: + CreateDatabaseEvent createDatabaseEvent = new CreateDatabaseEvent(null, status, null); + setEventId(true, createDatabaseEvent, eventId); + eventListener.onCreateDatabase(createDatabaseEvent); + break; + case DROP_DATABASE: + DropDatabaseEvent dropDatabaseEvent = new DropDatabaseEvent(null, status, null); + setEventId(true, dropDatabaseEvent, eventId); + eventListener.onDropDatabase(dropDatabaseEvent); + break; + case CREATE_TABLE: + + CreateTableEvent createTableEvent = new CreateTableEvent(null, status, null); + setEventId(true, createTableEvent, eventId); + eventListener.onCreateTable(createTableEvent); + break; + case DROP_TABLE: + DropTableEvent dropTableEvent = new DropTableEvent(null, status, false, null); + setEventId(true, dropTableEvent, eventId); + eventListener.onDropTable(dropTableEvent); + break; + default: + return eventId; + } + return eventId; + } + + private void verifyInvocations() throws Exception { + TSentryHmsEventNotification notification = new TSentryHmsEventNotification(); + int i = 1; + notification.setAuthorizable(new TSentryAuthorizable()); + + notification.setId(i); + notification.setEventType(EventMessage.EventType.CREATE_DATABASE.toString()); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + i += 2; + + + notification.setId(i); + notification.setEventType(EventMessage.EventType.CREATE_TABLE.toString()); + notification.setOwnerType(TSentryObjectOwnerType.USER); + Mockito.verify( + mockSentryClient, Mockito.times(1) + ).notifyHmsNotification(notification); + } + private void setEventId(boolean eventIdSet, ListenerEvent eventListener, long eventId) { if (eventIdSet) { eventListener.putParameter( - MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, String.valueOf(eventId)); + MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, String.valueOf(eventId)); } } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java index 3950ea5..dc1d67b 100644 --- a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java +++ b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClient.java @@ -292,4 +292,13 @@ public interface SentryPolicyServiceClient extends AutoCloseable { * @return The most recent processed notification ID. */ long syncNotifications(long id) throws SentryUserException; + + /** + * Notifies sentry server with the HMS Event and related metadata. + * @param sentryHmsEventNotification Event Notification message. + * @return The most recent processed notification ID. + */ + long notifyHmsNotification(TSentryHmsEventNotification sentryHmsEventNotification) throws SentryUserException; + + } http://git-wip-us.apache.org/repos/asf/sentry/blob/86c12d64/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java index 28d345e..5f76cb0 100644 --- a/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java +++ b/sentry-service/sentry-service-api/src/main/java/org/apache/sentry/api/service/thrift/SentryPolicyServiceClientDefaultImpl.java @@ -1167,4 +1167,15 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService throw new SentryUserException(THRIFT_EXCEPTION_MESSAGE, e); } } + + public long notifyHmsNotification(TSentryHmsEventNotification request) + throws SentryUserException { + try { + TSentryHmsEventNotificationResponse response = client.sentry_notify_hms_event(request); + Status.throwIfNotOk(response.getStatus()); + return response.getId(); + } catch (TException e) { + throw new SentryUserException(THRIFT_EXCEPTION_MESSAGE, e); + } + } } \ No newline at end of file
