SENTRY-1587 Change-Id: Ic21fdff7d7fd54d785683f0ec3dd3598cec33f74
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/96e1d9a1 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/96e1d9a1 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/96e1d9a1 Branch: refs/heads/sentry-ha-redesign-1 Commit: 96e1d9a1cc272c9c044f68034648300fcf9b8477 Parents: 18be1d5 Author: hahao <[email protected]> Authored: Fri Jan 27 15:00:36 2017 -0800 Committer: hahao <[email protected]> Committed: Fri Jan 27 15:00:36 2017 -0800 ---------------------------------------------------------------------- .../db/service/persistent/SentryStore.java | 62 +++++++++++++++++--- .../sentry/service/thrift/HMSFollower.java | 56 ++++++++++++++++-- .../db/service/persistent/TestSentryStore.java | 11 ++++ 3 files changed, 116 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index d1edcb1..9019350 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -2269,13 +2269,12 @@ public class SentryStore { return (PathsImage) tm.executeTransaction( new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { - long curChangeID = getLastProcessedPermChangeIDCore(pm); + + // curChangeID could be 0, since first full snapshot is fetching + // from HMS, and does not have corresponding delta update. + long curChangeID = getLastProcessedPathChangeIDCore(pm); Map<String, Set<String>> pathImage = retrieveFullPathsImageCore(pm); - if (curChangeID == EMPTY_CHANGE_ID && !pathImage.isEmpty()) { - throw new Exception("Non-empty full paths image should not have" + - "an empty change ID."); - } return new PathsImage(pathImage, curChangeID); } }); @@ -2331,7 +2330,7 @@ public class SentryStore { private void createAuthzPathsMappingCore(PersistenceManager pm, String authzObj, Set<String> paths) throws SentryAlreadyExistsException { - MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMapping(pm, authzObj); + MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMappingCore(pm, authzObj); if (mAuthzPathsMapping == null) { mAuthzPathsMapping = @@ -2342,10 +2341,59 @@ public class SentryStore { } } + + public void updateAuthzPathsMapping(final String hiveObj, final Set<String> paths) + throws Exception { + tm.executeTransactionWithRetry ( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + updateAuthzPathsMappingCore(pm, hiveObj, paths); + return null; + } + }); + } + + public void updateAuthzPathsMapping(final String hiveObj, final Set<String> paths, + final DeltaTransactionBlock deltaTransactionBlock) throws Exception { + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + updateAuthzPathsMappingCore(pm, hiveObj, paths); + return null; + } + }); + } + + private void updateAuthzPathsMappingCore(PersistenceManager pm, String authzObj, + Set<String> paths) throws SentryAlreadyExistsException { + + MAuthzPathsMapping mAuthzPathsMapping = getMAuthzPathsMappingCore(pm, authzObj); + + if (mAuthzPathsMapping == null) { + mAuthzPathsMapping = + new MAuthzPathsMapping(authzObj, paths, System.currentTimeMillis()); + } else { + mAuthzPathsMapping.getPaths().addAll(paths); + } + pm.makePersistent(mAuthzPathsMapping); + } + + /** + * Get the MAuthzPathsMapping object from authzObj + */ + public MAuthzPathsMapping getMAuthzPathsMapping(final String authzObj) + throws Exception{ + return tm.executeTransactionWithRetry( + new TransactionBlock<MAuthzPathsMapping>() { + public MAuthzPathsMapping execute(PersistenceManager pm) throws Exception { + return getMAuthzPathsMappingCore(pm, authzObj); + } + }); + } + /** * Get the MAuthzPathsMapping object from authzObj */ - public MAuthzPathsMapping getMAuthzPathsMapping(PersistenceManager pm, String authzObj) { + private MAuthzPathsMapping getMAuthzPathsMappingCore(PersistenceManager pm, String authzObj) { Query query = pm.newQuery(MAuthzPathsMapping.class); query.setFilter("this.authzObjName == t"); query.declareParameters("java.lang.String t"); http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index ad6bdda..7dce745 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -19,6 +19,7 @@ package org.apache.sentry.service.thrift; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -32,8 +33,8 @@ import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.core.common.exception.*; import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.PermissionsUpdate; -import org.apache.sentry.hdfs.UpdateableAuthzPaths; import org.apache.sentry.hdfs.FullUpdateInitializer; +import org.apache.sentry.hdfs.SentryMalformedPathException; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; @@ -321,12 +322,12 @@ public class HMSFollower implements Runnable { /** * Throws SentryInvalidHMSEventException if Notification event contains insufficient information */ - void processNotificationEvents(List<NotificationEvent> events) throws - SentryInvalidHMSEventException, SentryInvalidInputException { + void processNotificationEvents(List<NotificationEvent> events) throws Exception { SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); for (NotificationEvent event : events) { - String dbName, tableName, oldLocation, newLocation, location; + String dbName, tableName, oldLocation, newLocation, location, authzObj; + List<String> pathTree; switch (HCatEventMessage.EventType.valueOf(event.getEventType())) { case CREATE_DATABASE: SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage()); @@ -346,7 +347,15 @@ public class HMSFollower implements Runnable { throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e); } } - //TODO: HDFSPlugin.addPath(dbName, location) + // addPath into Sentry DB. Skip update if encounter malformed path. + authzObj = dbName; + pathTree = getPath(location); + if (pathTree != null) { + DeltaTransactionBlock deltaTransactionBlock = addPaths(authzObj, pathTree); + Set<String> paths = Sets.newHashSet(); + paths.add(PathsUpdate.getPath(pathTree)); + sentryStore.updateAuthzPathsMapping(authzObj, paths, deltaTransactionBlock); + } break; case DROP_DATABASE: SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage()); @@ -384,7 +393,15 @@ public class HMSFollower implements Runnable { throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e); } } - //TODO: HDFSPlugin.deletePath(dbName, location) + // addPath into Sentry DB. Skip update if encounter malformed path. + authzObj = dbName + "." + tableName; + pathTree = getPath(location); + if (pathTree != null) { + DeltaTransactionBlock deltaTransactionBlock = addPaths(authzObj, pathTree); + Set<String> paths = Sets.newHashSet(); + paths.add(PathsUpdate.getPath(pathTree)); + sentryStore.updateAuthzPathsMapping(authzObj, paths, deltaTransactionBlock); + } break; case DROP_TABLE: SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage()); @@ -522,4 +539,31 @@ public class HMSFollower implements Runnable { } return authzObj == null ? null : authzObj.toLowerCase(); } + + private DeltaTransactionBlock addPaths(String authzObj, List<String> pathTree) { + LOGGER.debug("#### HMS Path Update [" + + "OP : addPath, " + + "authzObj : " + authzObj.toLowerCase() + ", " + + "path : " + pathTree.toString() + "]"); + PathsUpdate update = new PathsUpdate(); + update.newPathChange(authzObj.toLowerCase()).addToAddPaths(pathTree); + return new DeltaTransactionBlock(update); + } + + /** + * Get path tree from a given path. It throws SentryMalformedPathException + * if encountered malformed path. + * + * @param path + * @return the path tree. + */ + private List<String> getPath(String path) { + try { + return PathsUpdate.parsePath(path); + } catch (SentryMalformedPathException e) { + LOGGER.error("Unexpected path = " + path); + e.printStackTrace(); + return null; + } + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/96e1d9a1/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index a26f996..a2d48a8 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -39,6 +39,7 @@ import org.apache.sentry.hdfs.PermissionsUpdate; import org.apache.sentry.hdfs.Updateable; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; +import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping; import org.apache.sentry.provider.db.service.model.MSentryPermChange; import org.apache.sentry.provider.db.service.model.MSentryPrivilege; import org.apache.sentry.provider.db.service.model.MSentryRole; @@ -2200,6 +2201,16 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1"), pathImage.get("db1.table1")); } + @Test + public void testUpdateAuthzPathsMapping() throws Exception { + sentryStore.updateAuthzPathsMapping("db1.table", Sets.newHashSet("/user/hive/warehouse/db1.db/table1")); + sentryStore.updateAuthzPathsMapping("db1.table", Sets.newHashSet("/user/hive/warehouse/db1.db/table2")); + + MAuthzPathsMapping mAuthzPathsMapping = sentryStore.getMAuthzPathsMapping("db1.table"); + assertEquals(Sets.newHashSet("/user/hive/warehouse/db1.db/table1", "/user/hive/warehouse/db1.db/table2"), + mAuthzPathsMapping.getPaths()); + } + public void testQueryParamBuilder() { SentryStore.QueryParamBuilder paramBuilder; paramBuilder = new SentryStore.QueryParamBuilder();
