Repository: sentry Updated Branches: refs/heads/master 67d64d0d5 -> 1a97f5ab0
SENTRY-1854: HMSFollower should handle notifications even if HDFS sync is disabled. (Na Li Reviewed by Vamsee Yarlagadda, Sergio Pena, kalyan kumar kalvagadda) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/1a97f5ab Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/1a97f5ab Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/1a97f5ab Branch: refs/heads/master Commit: 1a97f5ab0c222cdb5bc33461e4f934b530462871 Parents: 67d64d0 Author: Kalyan Kumar Kalvagadda <kkal...@cloudera.com> Authored: Mon Aug 7 17:10:50 2017 -0500 Committer: Kalyan Kumar Kalvagadda <kkal...@cloudera.com> Committed: Mon Aug 7 17:10:50 2017 -0500 ---------------------------------------------------------------------- .../service/persistent/DelegateSentryStore.java | 1 + .../db/service/persistent/SentryStore.java | 57 ++++++++------ .../db/service/thrift/SentryAdminServlet.java | 1 + .../sentry/service/thrift/HMSFollower.java | 9 ++- .../service/thrift/NotificationProcessor.java | 66 +++++++++++++--- .../sentry/service/thrift/SentryService.java | 14 ++-- .../service/thrift/SentryServiceUtil.java | 67 ++++++++++++++-- .../TestHMSFollowerSentryStoreIntegration.java | 13 +-- .../db/service/persistent/TestSentryStore.java | 74 +++++++++++++++++ .../persistent/TestSentryStoreImportExport.java | 3 + .../service/persistent/TestSentryVersion.java | 6 ++ .../sentry/service/thrift/TestHMSFollower.java | 83 ++++++++++++++++++++ .../thrift/TestNotificationProcessor.java | 20 +++++ 13 files changed, 350 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java index 4cb46ab..c221c34 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java @@ -40,6 +40,7 @@ import org.apache.sentry.provider.db.service.persistent.TransactionBlock; import org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor; import org.apache.sentry.provider.db.service.thrift.TSentryGroup; import org.apache.sentry.provider.db.service.thrift.TSentryRole; +import org.apache.sentry.service.thrift.SentryServiceUtil; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import com.google.common.annotations.VisibleForTesting; http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index f4842a9..d20de26 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -38,7 +38,6 @@ import javax.jdo.JDOHelper; import javax.jdo.PersistenceManager; import javax.jdo.PersistenceManagerFactory; import javax.jdo.Query; -import javax.jdo.Transaction; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -172,6 +171,10 @@ public class SentryStore { private Configuration conf; private final TransactionManager tm; + // When it is true, execute DeltaTransactionBlock to persist delta changes. + // When it is false, do not execute DeltaTransactionBlock + private boolean persistUpdateDeltas; + /** * counterWait is used to synchronize notifications between Thrift and HMSFollower. * Technically it doesn't belong here, but the only thing that connects HMSFollower @@ -263,6 +266,11 @@ public class SentryStore { verifySentryStoreSchema(checkSchemaVersion); } + public void setPersistUpdateDeltas(boolean persistUpdateDeltas) { + this.persistUpdateDeltas = persistUpdateDeltas; + } + + public TransactionManager getTransactionManager() { return tm; } @@ -730,7 +738,7 @@ public class SentryStore { final String roleName, final TSentryPrivilege privilege, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects String trimmedRoleName = trimAndLower(roleName); @@ -889,7 +897,7 @@ public class SentryStore { private synchronized void alterSentryRoleRevokePrivilege(final String grantorPrincipal, final String roleName, final TSentryPrivilege tPrivilege, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects String trimmedRoleName = safeTrimLower(roleName); @@ -1224,8 +1232,7 @@ public class SentryStore { */ public synchronized void dropSentryRole(final String roleName, final Update update) throws Exception { - - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects dropSentryRoleCore(pm, roleName); @@ -1303,7 +1310,7 @@ public class SentryStore { final String roleName, final Set<TSentryGroup> groupNames, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects alterSentryRoleAddGroupsCore(pm, roleName, groupNames); @@ -1452,7 +1459,7 @@ public class SentryStore { public synchronized void alterSentryRoleDeleteGroups(final String roleName, final Set<TSentryGroup> groupNames, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects String trimmedRoleName = trimAndLower(roleName); @@ -2189,8 +2196,7 @@ public class SentryStore { */ public synchronized void dropPrivilege(final TSentryAuthorizable tAuthorizable, final Update update) throws Exception { - - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects @@ -2269,7 +2275,7 @@ public class SentryStore { final TSentryAuthorizable newTAuthorizable, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects @@ -2753,7 +2759,7 @@ public class SentryStore { */ public void addAuthzPathsMapping(final String authzObj, final Collection<String> paths, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects addAuthzPathsMappingCore(pm, authzObj, paths); @@ -2800,7 +2806,7 @@ public class SentryStore { */ public void deleteAuthzPathsMapping(final String authzObj, final Iterable<String> paths, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects deleteAuthzPathsMappingCore(pm, authzObj, paths); @@ -2852,7 +2858,7 @@ public class SentryStore { */ public void deleteAllAuthzPathsMapping(final String authzObj, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects deleteAllAuthzPathsMappingCore(pm, authzObj); @@ -2901,7 +2907,7 @@ public class SentryStore { */ public void renameAuthzPathsMapping(final String oldObj, final String newObj, final String oldPath, final String newPath, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects renameAuthzPathsMappingCore(pm, oldObj, newObj, oldPath, newPath); @@ -2958,7 +2964,7 @@ public class SentryStore { */ public void renameAuthzObj(final String oldObj, final String newObj, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects renameAuthzObjCore(pm, oldObj, newObj); @@ -3041,7 +3047,7 @@ public class SentryStore { */ public void updateAuthzPathsMapping(final String authzObj, final String oldPath, final String newPath, final Update update) throws Exception { - execute(new DeltaTransactionBlock(update), new TransactionBlock<Object>() { + execute(update, new TransactionBlock<Object>() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects updateAuthzPathsMappingCore(pm, authzObj, oldPath, newPath); @@ -4080,7 +4086,8 @@ public class SentryStore { } // The first delta change from the DB should match the changeID - // If it doesn't, then it means the delta table no longer has entries starting from the requested CHANGE_ID + // If it doesn't, then it means the delta table no longer has entries starting from the + // requested CHANGE_ID if (changes.get(0).getChangeID() != changeID) { LOGGER.debug(String.format("Starting delta change from %s is off from the requested id. " + "Requested changeID: %s, Missing delta count: %s", @@ -4101,24 +4108,26 @@ public class SentryStore { } /** - * Execute Perm/Path UpdateTransaction and corresponding actual - * action transaction, e.g dropSentryRole, in a single transaction. + * Execute actual Perm/Path action transaction, e.g dropSentryRole, and persist corresponding + * Update in a single transaction if persistUpdateDeltas is true. * Note that this method only applies to TransactionBlock that * does not have any return value. * <p> * Failure in any TransactionBlock would cause the whole transaction * to fail. * - * @param deltaTransactionBlock + * @param update * @param transactionBlock * @throws Exception */ - private void execute(DeltaTransactionBlock deltaTransactionBlock, + private void execute(Update update, TransactionBlock<Object> transactionBlock) throws Exception { - List<TransactionBlock<Object>> tbs = Lists.newArrayList(); - if (deltaTransactionBlock != null) { - tbs.add(deltaTransactionBlock); + List<TransactionBlock<Object>> tbs = new ArrayList(2); + + if (persistUpdateDeltas) { + tbs.add(new DeltaTransactionBlock(update)); } + tbs.add(transactionBlock); tm.executeTransactionBlocksWithRetry(tbs); } http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryAdminServlet.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryAdminServlet.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryAdminServlet.java index 8a8bbd3..17dc13f 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryAdminServlet.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryAdminServlet.java @@ -31,6 +31,7 @@ import java.io.Writer; import java.util.HashMap; import java.util.Map; import java.util.Set; +import org.apache.sentry.service.thrift.SentryServiceUtil; /** * Admin Servlet is only used when SENTRY_WEB_ADMIN_SERVLET_ENABLED is true. http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index 9e8e0e7..b0a202e 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -48,6 +48,7 @@ public class HMSFollower implements Runnable, AutoCloseable { private final Configuration authzConf; private final SentryStore sentryStore; private final NotificationProcessor notificationProcessor; + private final boolean hdfsSyncEnabled; private final LeaderStatusMonitor leaderMonitor; @@ -86,6 +87,7 @@ public class HMSFollower implements Runnable, AutoCloseable { notificationProcessor = new NotificationProcessor(sentryStore, authServerName, authzConf); client = new SentryHMSClient(authzConf, hiveConnectionFactory); + hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabledNoCache(authzConf); // no cache to test different settings for hdfs sync } @VisibleForTesting @@ -285,7 +287,12 @@ public class HMSFollower implements Runnable, AutoCloseable { try { LOGGER.debug("Persisting HMS path full snapshot"); - sentryStore.persistFullPathsImage(snapshotInfo.getPathImage()); + + if (hdfsSyncEnabled) { + sentryStore.persistFullPathsImage(snapshotInfo.getPathImage()); + } + + // We need to persist latest notificationID for next poll sentryStore.persistLastProcessedNotificationID(snapshotInfo.getId()); } catch (Exception failure) { LOGGER.error("Received exception while persisting HMS path full snapshot "); http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java index f631869..9e28da4 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java @@ -78,6 +78,7 @@ final class NotificationProcessor { // These variables can be updated even after object is instantiated, for testing purposes. private boolean syncStoreOnCreate = false; private boolean syncStoreOnDrop = false; + private final boolean hdfsSyncEnabled; /** * Configuring notification processor. @@ -96,6 +97,7 @@ final class NotificationProcessor { AUTHZ_SYNC_CREATE_WITH_POLICY_STORE.getDefault())); syncStoreOnDrop = Boolean.parseBoolean(conf.get(AUTHZ_SYNC_DROP_WITH_POLICY_STORE.getVar(), AUTHZ_SYNC_DROP_WITH_POLICY_STORE.getDefault())); + hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabled(conf); } /** @@ -241,12 +243,19 @@ final class NotificationProcessor { StringUtils.defaultIfBlank(location, "null")); return false; } - List<String> locations = Collections.singletonList(location); - addPaths(dbName, locations, event.getEventId()); + if (syncStoreOnCreate) { dropSentryDbPrivileges(dbName, event); } - return true; + + if (hdfsSyncEnabled) { + List<String> locations = Collections.singletonList(location); + addPaths(dbName, locations, event.getEventId()); + + return true; + } + + return false; } /** @@ -268,9 +277,13 @@ final class NotificationProcessor { if (syncStoreOnDrop) { dropSentryDbPrivileges(dbName, event); } - List<String> locations = Collections.singletonList(location); - removePaths(dbName, locations, event.getEventId()); - return true; + + if (hdfsSyncEnabled) { + List<String> locations = Collections.singletonList(location); + removePaths(dbName, locations, event.getEventId()); + return true; + } + return false; } /** @@ -298,10 +311,15 @@ final class NotificationProcessor { if (syncStoreOnCreate) { dropSentryTablePrivileges(dbName, tableName, event); } - String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); - List<String> locations = Collections.singletonList(location); - addPaths(authzObj, locations, event.getEventId()); - return true; + + if (hdfsSyncEnabled) { + String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); + List<String> locations = Collections.singletonList(location); + addPaths(authzObj, locations, event.getEventId()); + return true; + } + + return false; } /** @@ -327,9 +345,14 @@ final class NotificationProcessor { if (syncStoreOnDrop) { dropSentryTablePrivileges(dbName, tableName, event); } - String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); - removeAllPaths(authzObj, event.getEventId()); - return true; + + if (hdfsSyncEnabled) { + String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); + removeAllPaths(authzObj, event.getEventId()); + return true; + } + + return false; } /** @@ -340,6 +363,11 @@ final class NotificationProcessor { * @throws Exception if encounters errors while persisting the path change */ private boolean processAlterTable(NotificationEvent event) throws Exception { + + if (!hdfsSyncEnabled) { + return false; + } + SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(event.getMessage()); String oldDbName = alterTableMessage.getDB(); @@ -404,6 +432,10 @@ final class NotificationProcessor { */ private boolean processAddPartition(NotificationEvent event) throws Exception { + if (!hdfsSyncEnabled) { + return false; + } + SentryJSONAddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(event.getMessage()); String dbName = addPartitionMessage.getDB(); @@ -431,6 +463,10 @@ final class NotificationProcessor { */ private boolean processDropPartition(NotificationEvent event) throws Exception { + if (!hdfsSyncEnabled) { + return false; + } + SentryJSONDropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(event.getMessage()); String dbName = dropPartitionMessage.getDB(); @@ -457,6 +493,10 @@ final class NotificationProcessor { * @throws Exception if encounters errors while persisting the path change */ private boolean processAlterPartition(NotificationEvent event) throws Exception { + if (!hdfsSyncEnabled) { + return false; + } + SentryJSONAlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(event.getMessage()); String dbName = alterPartitionMessage.getDB(); http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java index 10d55dc..d44abff 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java @@ -113,7 +113,6 @@ public class SentryService implements Callable, SigUtils.SigListener { private final SentryStore sentryStore; private ScheduledExecutorService sentryStoreCleanService; private final LeaderStatusMonitor leaderMonitor; - private final boolean hdfsSyncEnabled; public SentryService(Configuration conf) throws Exception { this.conf = conf; @@ -163,11 +162,10 @@ public class SentryService implements Callable, SigUtils.SigListener { .build(); serviceExecutor = Executors.newSingleThreadExecutor(sentryServiceThreadFactory); this.sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(SentryServiceUtil.isHDFSSyncEnabled(conf)); this.leaderMonitor = LeaderStatusMonitor.getLeaderStatusMonitor(conf); webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT); - hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabled(conf); - status = Status.NOT_STARTED; // Enable signal handler for HA leader/follower status if configured @@ -276,8 +274,10 @@ public class SentryService implements Callable, SigUtils.SigListener { } private void startHMSFollower(Configuration conf) throws Exception { - if (!hdfsSyncEnabled) { - LOGGER.info("HMS follower is not started because HDFS sync is disabled."); + boolean syncPolicyStore = SentryServiceUtil.isSyncPolicyStoreEnabled(conf); + + if ((!SentryServiceUtil.isHDFSSyncEnabled(conf)) && (!syncPolicyStore)) { + LOGGER.info("HMS follower is not started because HDFS sync is disabled and perm sync is disabled"); return; } @@ -315,10 +315,6 @@ public class SentryService implements Callable, SigUtils.SigListener { } private void stopHMSFollower(Configuration conf) { - if (!hdfsSyncEnabled) { - return; - } - if ((hmsFollowerExecutor == null) || (hmsFollower == null)) { Preconditions.checkState(hmsFollower == null); Preconditions.checkState(hmsFollowerExecutor == null); http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java index 5826766..3488e11 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java @@ -29,6 +29,10 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_ALTER_WITH_POLICY_STORE; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE; + import org.apache.hadoop.hive.conf.HiveConf; import org.apache.sentry.core.common.exception.SentryInvalidInputException; import org.apache.sentry.core.common.utils.SentryConstants; @@ -45,6 +49,9 @@ import org.slf4j.Logger; public final class SentryServiceUtil { + private static boolean firstCallHDFSSyncEnabled = true; + private static boolean hdfsSyncEnabled = false; + // parse the privilege in String and get the TSentryPrivilege as result public static TSentryPrivilege convertToTSentryPrivilege(String privilegeStr) { TSentryPrivilege tSentryPrivilege = new TSentryPrivilege(); @@ -200,21 +207,71 @@ public final class SentryServiceUtil { } /** - * Checks if Sentry is configured with HDFS sync enabled. + * Check if Sentry is configured with HDFS sync enabled. Cache the result * * @param conf The Configuration object where HDFS sync configurations are set. * @return True if enabled; False otherwise. */ - static boolean isHDFSSyncEnabled(Configuration conf) { + public static boolean isHDFSSyncEnabled(Configuration conf) { + if (firstCallHDFSSyncEnabled) { + List<String> processorFactories = + Arrays.asList(conf.get(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "").split(",")); + + List<String> policyStorePlugins = + Arrays.asList( + conf.get(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "").split(",")); + + hdfsSyncEnabled = + processorFactories.contains("org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory") + && policyStorePlugins.contains("org.apache.sentry.hdfs.SentryPlugin"); + firstCallHDFSSyncEnabled = false; + } + + return hdfsSyncEnabled; + } + + /** + * Check if Sentry is configured with HDFS sync enabled without caching the result + * + * @param conf The Configuration object where HDFS sync configurations are set. + * @return True if enabled; False otherwise. + */ + public static boolean isHDFSSyncEnabledNoCache(Configuration conf) { + List<String> processorFactories = Arrays.asList(conf.get(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "").split(",")); List<String> policyStorePlugins = - Arrays.asList(conf.get(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "").split(",")); + Arrays.asList( + conf.get(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "").split(",")); + + hdfsSyncEnabled = + processorFactories.contains("org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory") + && policyStorePlugins.contains("org.apache.sentry.hdfs.SentryPlugin"); + + + return hdfsSyncEnabled; + } + + /** + * Check if Sentry is configured with policy store sync enabled + * @param conf + * @return True if enabled; False otherwise + */ + public static boolean isSyncPolicyStoreEnabled(Configuration conf) { + boolean syncStoreOnCreate; + boolean syncStoreOnDrop; + boolean syncStoreOnAlter; + syncStoreOnCreate = Boolean + .parseBoolean(conf.get(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE.getVar(), + AUTHZ_SYNC_CREATE_WITH_POLICY_STORE.getDefault())); + syncStoreOnDrop = Boolean.parseBoolean(conf.get(AUTHZ_SYNC_DROP_WITH_POLICY_STORE.getVar(), + AUTHZ_SYNC_DROP_WITH_POLICY_STORE.getDefault())); + syncStoreOnAlter = Boolean.parseBoolean(conf.get(AUTHZ_SYNC_ALTER_WITH_POLICY_STORE.getVar(), + AUTHZ_SYNC_ALTER_WITH_POLICY_STORE.getDefault())); - return processorFactories.contains("org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory") - && policyStorePlugins.contains("org.apache.sentry.hdfs.SentryPlugin"); + return syncStoreOnCreate || syncStoreOnDrop || syncStoreOnAlter; } static String getHiveMetastoreURI() { http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestHMSFollowerSentryStoreIntegration.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestHMSFollowerSentryStoreIntegration.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestHMSFollowerSentryStoreIntegration.java index 82f600b..4ee1597 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestHMSFollowerSentryStoreIntegration.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestHMSFollowerSentryStoreIntegration.java @@ -82,7 +82,6 @@ public class TestHMSFollowerSentryStoreIntegration { // THis should be a UserGroupInformation provider CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0); - // The user credentials are stored as a static variable by UserGrouoInformation provider. // We need to only set the password the first time, an attempt to set it for the second // time fails with an exception. @@ -109,6 +108,7 @@ public class TestHMSFollowerSentryStoreIntegration { @Before public void before() throws Exception { sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(true); policyFile = new PolicyFile(); String adminUser = "g1"; addGroupsToUser(adminUser, adminGroups); @@ -155,17 +155,6 @@ public class TestHMSFollowerSentryStoreIntegration { sentryStore.getMSentryRoleByName(roleName).getRoleName()); } - /** - * Create a role with the given name and verify that it is created - * @param roleName - * @throws Exception - */ - private void createRole(String roleName) throws Exception { - checkRoleDoesNotExist(roleName); - sentryStore.createSentryRole(roleName); - checkRoleExists(roleName); - } - private TSentryAuthorizable toTSentryAuthorizable( TSentryPrivilege tSentryPrivilege) { TSentryAuthorizable tSentryAuthorizable = new TSentryAuthorizable(); http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index a8ebf7c..859a9fe 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -59,6 +59,8 @@ import org.apache.sentry.provider.db.service.thrift.TSentryGroup; import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; import org.apache.sentry.provider.db.service.thrift.TSentryRole; import org.apache.sentry.provider.file.PolicyFile; +import org.apache.sentry.service.thrift.SentryServiceUtil; +import org.apache.sentry.service.thrift.ServiceConstants; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.After; import org.junit.AfterClass; @@ -96,6 +98,10 @@ public class TestSentryStore extends org.junit.Assert { final String ourUrl = UserProvider.SCHEME_NAME + ":///"; conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + // enable HDFS sync, so perm and path changes will be saved into DB + conf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + conf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + // THis should be a UserGroupInformation provider CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0); @@ -120,7 +126,9 @@ public class TestSentryStore extends org.junit.Assert { conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFilePath.getPath()); conf.setInt(ServerConfig.SENTRY_STORE_TRANSACTION_RETRY, 10); + boolean hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabled(conf); sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(hdfsSyncEnabled); } @Before @@ -3244,4 +3252,70 @@ public class TestSentryStore extends org.junit.Assert { sentryStore.clearAllTables(); assertEquals(sentryStore.isAuthzPathsMappingEmpty(), true); } + + @Test + public void testAddDeleteAuthzPathsMappingNoDeltaSavedWithoutHDFSSync() throws Exception { + + // disable HDFS + conf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, ""); + conf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, ""); + SentryStore localSentryStore = new SentryStore(conf); + + // Persist an empty image so that we can add paths to it. + localSentryStore.persistFullPathsImage(new HashMap<String, Set<String>>()); + + // Add "db1.table1" authzObj + Long lastNotificationId = sentryStore.getLastProcessedNotificationID(); + PathsUpdate addUpdate = new PathsUpdate(1, false); + addUpdate.newPathChange("db1.table"). + addToAddPaths(Arrays.asList("db1", "tbl1")); + addUpdate.newPathChange("db1.table"). + addToAddPaths(Arrays.asList("db1", "tbl2")); + + localSentryStore.addAuthzPathsMapping("db1.table", + Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate); + PathsImage pathsImage = localSentryStore.retrieveFullPathsImage(); + Map<String, Set<String>> pathImage = pathsImage.getPathImage(); + assertEquals(1, pathImage.size()); + assertEquals(2, pathImage.get("db1.table").size()); + assertEquals(2, localSentryStore.getMPaths().size()); + + // Query the persisted path change and ensure it is not saved + long lastChangeID = localSentryStore.getLastProcessedPathChangeID(); + assertEquals(0, lastChangeID); + + // Delete path 'db1.db/tbl1' from "db1.table1" authzObj. + PathsUpdate delUpdate = new PathsUpdate(2, false); + delUpdate.newPathChange("db1.table") + .addToDelPaths(Arrays.asList("db1", "tbl1")); + localSentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate); + pathImage = localSentryStore.retrieveFullPathsImage().getPathImage(); + assertEquals(1, pathImage.size()); + assertEquals(1, pathImage.get("db1.table").size()); + assertEquals(1, localSentryStore.getMPaths().size()); + + // Query the persisted path change and ensure it is not saved + lastChangeID = localSentryStore.getLastProcessedPathChangeID(); + assertEquals(0, lastChangeID); + + // Delete "db1.table" authzObj from the authzObj -> [Paths] mapping. + PathsUpdate delAllupdate = new PathsUpdate(3, false); + delAllupdate.newPathChange("db1.table") + .addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS)); + localSentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate); + pathImage = localSentryStore.retrieveFullPathsImage().getPathImage(); + assertEquals(0, pathImage.size()); + assertEquals(0, localSentryStore.getMPaths().size()); + + // Query the persisted path change and ensure it is not saved + lastChangeID = localSentryStore.getLastProcessedPathChangeID(); + assertEquals(0, lastChangeID); + + lastNotificationId = localSentryStore.getLastProcessedNotificationID(); + assertEquals(0, lastNotificationId.longValue()); + + // enable HDFS for other tests + conf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + conf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStoreImportExport.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStoreImportExport.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStoreImportExport.java index 1c3a4f2..b085ac3 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStoreImportExport.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStoreImportExport.java @@ -37,6 +37,7 @@ import org.apache.sentry.provider.db.service.thrift.TSentryGrantOption; import org.apache.sentry.provider.db.service.thrift.TSentryMappingData; import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; import org.apache.sentry.provider.file.PolicyFile; +import org.apache.sentry.service.thrift.SentryServiceUtil; import org.apache.sentry.service.thrift.ServiceConstants.PrivilegeScope; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.After; @@ -79,7 +80,9 @@ public class TestSentryStoreImportExport { policyFilePath = new File(dataDir, "local_policy_file.ini"); conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFilePath.getPath()); policyFile = new PolicyFile(); + boolean hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabled(conf); sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(hdfsSyncEnabled); String adminUser = "g1"; addGroupsToUser(adminUser, adminGroups); http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryVersion.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryVersion.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryVersion.java index a8e8a03..2b7dcde 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryVersion.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryVersion.java @@ -24,6 +24,7 @@ import java.io.File; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; +import org.apache.sentry.service.thrift.SentryServiceUtil; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.junit.Before; import org.junit.Test; @@ -53,10 +54,13 @@ public class TestSentryVersion { @Test public void testVerifySentryVersionCheck() throws Exception { conf.set(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false"); + boolean hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabled(conf); SentryStore sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(hdfsSyncEnabled); sentryStore.stop(); conf.set(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "true"); sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(hdfsSyncEnabled); } /** @@ -77,7 +81,9 @@ public class TestSentryVersion { @Test public void testSentryImplicitVersion() throws Exception { conf.set(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false"); + boolean hdfsSyncEnabled = SentryServiceUtil.isHDFSSyncEnabled(conf); SentryStore sentryStore = new SentryStore(conf); + sentryStore.setPersistUpdateDeltas(hdfsSyncEnabled); assertEquals(SentryStoreSchemaInfo.getSentryVersion(), sentryStore.getSentryVersion()); } http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java index 23f8de8..6820a9b 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java @@ -66,6 +66,10 @@ public class TestHMSFollower { hiveConnectionFactory = new HiveSimpleConnectionFactory(configuration, new HiveConf()); hiveConnectionFactory.init(); configuration.set("sentry.hive.sync.create", "true"); + + // enable HDFS sync, so perm and path changes will be saved into DB + configuration.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + configuration.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); } @Test @@ -606,6 +610,10 @@ public class TestHMSFollower { Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); Configuration configuration = new Configuration(); + // enable HDFS sync, so perm and path changes will be saved into DB + configuration.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + configuration.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, null, hiveConnectionFactory, hiveInstance); @@ -739,4 +747,79 @@ public class TestHMSFollower { verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); } + + @Test + public void testNoHdfsNoPersistAFullSnapshot() throws Exception { + /* + * TEST CASE + * + * Simulates (by using mocks) that Sentry has not processed any notifications, so this + * should trigger a new full HMS snapshot request with the eventId = 1 + */ + + final long SENTRY_PROCESSED_EVENT_ID = SentryStore.EMPTY_NOTIFICATION_ID; + final long HMS_PROCESSED_EVENT_ID = 1L; + + // Mock that returns a full snapshot + Map<String, Set<String>> snapshotObjects = new HashMap<>(); + snapshotObjects.put("db", Sets.newHashSet("/db")); + snapshotObjects.put("db.table", Sets.newHashSet("/db/table")); + PathsImage fullSnapshot = new PathsImage(snapshotObjects, HMS_PROCESSED_EVENT_ID, 1); + + SentryHMSClient sentryHmsClient = Mockito.mock(SentryHMSClient.class); + Mockito.when(sentryHmsClient.getFullSnapshot()).thenReturn(fullSnapshot); + Mockito.when(sentryHmsClient.getCurrentNotificationId()).thenReturn(fullSnapshot.getId()); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, null, + hiveConnectionFactory, hiveInstance); + hmsFollower.setSentryHmsClient(sentryHmsClient); + + // 1st run should get a full snapshot because AuthzPathsMapping is empty + Mockito.when(sentryStore.getLastProcessedNotificationID()).thenReturn(SENTRY_PROCESSED_EVENT_ID); + Mockito.when(sentryStore.isAuthzPathsMappingEmpty()).thenReturn(true); + hmsFollower.run(); + Mockito.verify(sentryStore, times(0)).persistFullPathsImage(fullSnapshot.getPathImage()); + Mockito.verify(sentryStore, times(1)).persistLastProcessedNotificationID(fullSnapshot.getId()); + } + + @Test + public void testNoHdfsSyncAlterTableNotPersisted() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + String newDbName = "db1"; + String newTableName = "table2"; + + // Create notification events + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs:///db1.db/table1"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.ALTER_TABLE.toString(), + messageFactory.buildAlterTableMessage( + new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null), + new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null)) + .toString()); + notificationEvent.setDbName(newDbName); + notificationEvent.setTableName(newTableName); + List<NotificationEvent> events = new ArrayList<>(); + events.add(notificationEvent); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, null, + hiveConnectionFactory, hiveInstance); + hmsFollower.processNotifications(events); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + + TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + newAuthorizable.setDb(newDbName); + newAuthorizable.setTable(newTableName); + + verify(sentryStore, times(0)).renamePrivilege(authorizable, newAuthorizable, + NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/1a97f5ab/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java index c6c9448..0fd4ff0 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java @@ -54,6 +54,10 @@ public class TestNotificationProcessor { public static void setup() { conf.set("sentry.hive.sync.create", "true"); conf.set("sentry.hive.sync.drop", "true"); + + // enable HDFS sync, so perm and path changes will be saved into DB + conf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + conf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); } @After @@ -253,6 +257,10 @@ public class TestNotificationProcessor { String tableName = "table1"; Configuration authConf = new Configuration(); + // enable HDFS sync, so perm and path changes will be saved into DB + authConf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + authConf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + notificationProcessor = new NotificationProcessor(sentryStore, hiveInstance, authConf); @@ -291,6 +299,10 @@ public class TestNotificationProcessor { String newTableName = "table2"; Configuration authConf = new Configuration(); + // enable HDFS sync, so perm and path changes will be saved into DB + authConf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + authConf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + notificationProcessor = new NotificationProcessor(sentryStore, hiveInstance, authConf); @@ -338,6 +350,10 @@ public class TestNotificationProcessor { String newTableName = "table2"; Configuration authConf = new Configuration(); + // enable HDFS sync, so perm and path changes will be saved into DB + authConf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + authConf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + notificationProcessor = new NotificationProcessor(sentryStore, hiveInstance, authConf); @@ -393,6 +409,10 @@ public class TestNotificationProcessor { Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); Configuration authConf = new Configuration(); + // enable HDFS sync, so perm and path changes will be saved into DB + authConf.set(ServiceConstants.ServerConfig.PROCESSOR_FACTORIES, "org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"); + authConf.set(ServiceConstants.ServerConfig.SENTRY_POLICY_STORE_PLUGINS, "org.apache.sentry.hdfs.SentryPlugin"); + notificationProcessor = new NotificationProcessor(sentryStore, hiveInstance, authConf);