SENTRY-1536 Change-Id: I1d7d21891692c3dd7557b4e5f90d9b0c35d820bd
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/ee2d3f7a Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/ee2d3f7a Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/ee2d3f7a Branch: refs/heads/sentry-ha-redesign-1 Commit: ee2d3f7a2ebc409f74f7de07fd316fe74f03dd11 Parents: d5176b2 Author: hahao <[email protected]> Authored: Thu Jan 26 17:29:39 2017 -0800 Committer: hahao <[email protected]> Committed: Thu Jan 26 17:29:39 2017 -0800 ---------------------------------------------------------------------- .../org/apache/sentry/hdfs/PathsUpdate.java | 26 + .../apache/sentry/hdfs/PermissionsUpdate.java | 27 + .../sentry/hdfs/UpdateableAuthzPermissions.java | 1 + .../org/apache/sentry/hdfs/SentryPlugin.java | 87 ++- .../provider/db/SentryPolicyStorePlugin.java | 29 +- .../service/persistent/DelegateSentryStore.java | 79 +-- .../db/service/model/MSentryPathChange.java | 30 +- .../db/service/model/MSentryPermChange.java | 28 +- .../provider/db/service/model/package.jdo | 4 +- .../persistent/DeltaTransactionBlock.java | 105 ++++ .../db/service/persistent/SentryStore.java | 549 +++++++++++++++++-- .../service/persistent/TransactionManager.java | 95 +++- .../thrift/SentryPolicyStoreProcessor.java | 103 ++-- .../sentry/service/thrift/HMSFollower.java | 49 +- .../db/service/persistent/TestSentryStore.java | 209 +++++++ .../persistent/TestSentryStoreImportExport.java | 2 +- .../sentry/service/thrift/TestHMSFollower.java | 13 +- 17 files changed, 1219 insertions(+), 217 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index 9ecd9e4..ffb0756 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -173,4 +173,30 @@ public class PathsUpdate implements Updateable.Update { return ThriftSerializer.serializeToJSON(tPathsUpdate); } + @Override + public int hashCode() { + return (tPathsUpdate == null) ? 0 : tPathsUpdate.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + PathsUpdate other = (PathsUpdate) obj; + if (tPathsUpdate == null) { + return other.tPathsUpdate == null; + } + return tPathsUpdate.equals(other.tPathsUpdate); + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java index a346587..14a4a0f 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PermissionsUpdate.java @@ -117,4 +117,31 @@ public class PermissionsUpdate implements Updateable.Update { public String JSONSerialize() throws TException { return ThriftSerializer.serializeToJSON(tPermUpdate); } + + @Override + public int hashCode() { + return (tPermUpdate == null) ? 0 : tPermUpdate.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + PermissionsUpdate other = (PermissionsUpdate) obj; + if (tPermUpdate == null) { + return other.tPermUpdate == null; + } + return tPermUpdate.equals(other.tPermUpdate); + } + } http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java index 2472928..766611c 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/UpdateableAuthzPermissions.java @@ -217,6 +217,7 @@ public class UpdateableAuthzPermissions implements AuthzPermissions, Updateable< @Override public PermissionsUpdate createFullImageUpdate(long currSeqNum) { + // Using in-memory cache perms to create a full permission snapshot. PermissionsUpdate retVal = new PermissionsUpdate(currSeqNum, true); for (PrivilegeInfo pInfo : perms.getAllPrivileges()) { TPrivilegeChanges pUpdate = retVal.addPrivilegeUpdate(pInfo.getAuthzObj()); http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index 47c9f9d..4c3e9d6 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -34,6 +34,7 @@ import org.apache.sentry.hdfs.service.thrift.TPermissionsUpdate; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; import org.apache.sentry.provider.db.SentryPolicyStorePlugin; +import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest; @@ -162,12 +163,13 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen private UpdateForwarder<PathsUpdate> pathsUpdater; private UpdateForwarder<PermissionsUpdate> permsUpdater; + // TODO: Each perm change sequence number should be generated during persistence at sentry store. private final AtomicLong permSeqNum = new AtomicLong(5); private PermImageRetriever permImageRetriever; private boolean outOfSync = false; /* * This number is smaller than starting sequence numbers used by NN and HMS - * so in both cases its effect is to creat appearence of out-of-sync + * so in both cases its effect is to create appearance of out-of-sync * updates on the Sentry server (as if there were no previous updates at all). * It, in turn, triggers a) pushing full update from HMS to Sentry and * b) pulling full update from Sentry to NameNode. @@ -287,19 +289,21 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen } @Override - public void onAlterSentryRoleAddGroups( + public DeltaTransactionBlock onAlterSentryRoleAddGroups( TAlterSentryRoleAddGroupsRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); TRoleChanges rUpdate = update.addRoleUpdate(request.getRoleName()); for (TSentryGroup group : request.getGroups()) { rUpdate.addToAddGroups(group.getGroupName()); } + permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); + return new DeltaTransactionBlock(update); } @Override - public void onAlterSentryRoleDeleteGroups( + public DeltaTransactionBlock onAlterSentryRoleDeleteGroups( TAlterSentryRoleDeleteGroupsRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); @@ -307,38 +311,50 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen for (TSentryGroup group : request.getGroups()) { rUpdate.addToDelGroups(group.getGroupName()); } + permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); + return new DeltaTransactionBlock(update); } @Override - public void onAlterSentryRoleGrantPrivilege( - TAlterSentryRoleGrantPrivilegeRequest request) - throws SentryPluginException { + public void onAlterSentryRoleGrantPrivilege(TAlterSentryRoleGrantPrivilegeRequest request, + Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap) throws SentryPluginException { + if (request.isSetPrivileges()) { String roleName = request.getRoleName(); + for (TSentryPrivilege privilege : request.getPrivileges()) { if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) { - onAlterSentryRoleGrantPrivilegeCore(roleName, privilege); + PermissionsUpdate update = onAlterSentryRoleGrantPrivilegeCore(roleName, privilege); + DeltaTransactionBlock tb = new DeltaTransactionBlock(update); + + if (update != null && privilegesUpdateMap != null) { + privilegesUpdateMap.put(privilege, tb); + } } } } } - private void onAlterSentryRoleGrantPrivilegeCore(String roleName, TSentryPrivilege privilege) + private PermissionsUpdate onAlterSentryRoleGrantPrivilegeCore(String roleName, TSentryPrivilege privilege) throws SentryPluginException { String authzObj = getAuthzObj(privilege); - if (authzObj != null) { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - update.addPrivilegeUpdate(authzObj).putToAddPrivileges( - roleName, privilege.getAction().toUpperCase()); - permsUpdater.handleUpdateNotification(update); - LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + "].."); + if (authzObj == null) { + return null; } + + PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); + update.addPrivilegeUpdate(authzObj).putToAddPrivileges( + roleName, privilege.getAction().toUpperCase()); + + permsUpdater.handleUpdateNotification(update); + LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + "].."); + return update; } @Override - public void onRenameSentryPrivilege(TRenamePrivilegesRequest request) + public DeltaTransactionBlock onRenameSentryPrivilege(TRenamePrivilegesRequest request) throws SentryPluginException { String oldAuthz = getAuthzObj(request.getOldAuthorizable()); String newAuthz = getAuthzObj(request.getNewAuthorizable()); @@ -346,19 +362,28 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen TPrivilegeChanges privUpdate = update.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS); privUpdate.putToAddPrivileges(newAuthz, newAuthz); privUpdate.putToDelPrivileges(oldAuthz, oldAuthz); + permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + newAuthz + ", " + oldAuthz + "].."); + return new DeltaTransactionBlock(update); } @Override - public void onAlterSentryRoleRevokePrivilege( - TAlterSentryRoleRevokePrivilegeRequest request) + public void onAlterSentryRoleRevokePrivilege(TAlterSentryRoleRevokePrivilegeRequest request, + Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap) throws SentryPluginException { + if (request.isSetPrivileges()) { String roleName = request.getRoleName(); + for (TSentryPrivilege privilege : request.getPrivileges()) { if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) { - onAlterSentryRoleRevokePrivilegeCore(roleName, privilege); + PermissionsUpdate update = onAlterSentryRoleRevokePrivilegeCore(roleName, privilege); + DeltaTransactionBlock tb = new DeltaTransactionBlock(update); + + if (update != null && privilegesUpdateMap != null) { + privilegesUpdateMap.put(privilege, tb); + } } } } @@ -372,38 +397,46 @@ public class SentryPlugin implements SentryPolicyStorePlugin, SigUtils.SigListen this.outOfSync = outOfSync; } - private void onAlterSentryRoleRevokePrivilegeCore(String roleName, TSentryPrivilege privilege) + private PermissionsUpdate onAlterSentryRoleRevokePrivilegeCore(String roleName, TSentryPrivilege privilege) throws SentryPluginException { String authzObj = getAuthzObj(privilege); - if (authzObj != null) { - PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); - update.addPrivilegeUpdate(authzObj).putToDelPrivileges( - roleName, privilege.getAction().toUpperCase()); - permsUpdater.handleUpdateNotification(update); - LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); + if (authzObj == null) { + return null; } + + PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); + update.addPrivilegeUpdate(authzObj).putToDelPrivileges( + roleName, privilege.getAction().toUpperCase()); + + permsUpdater.handleUpdateNotification(update); + LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); + return update; } @Override - public void onDropSentryRole(TDropSentryRoleRequest request) + public DeltaTransactionBlock onDropSentryRole(TDropSentryRoleRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); update.addPrivilegeUpdate(PermissionsUpdate.ALL_AUTHZ_OBJ).putToDelPrivileges( request.getRoleName(), PermissionsUpdate.ALL_AUTHZ_OBJ); update.addRoleUpdate(request.getRoleName()).addToDelGroups(PermissionsUpdate.ALL_GROUPS); + permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + request.getRoleName() + "].."); + return new DeltaTransactionBlock(update); } @Override - public void onDropSentryPrivilege(TDropPrivilegesRequest request) + public DeltaTransactionBlock onDropSentryPrivilege(TDropPrivilegesRequest request) throws SentryPluginException { PermissionsUpdate update = new PermissionsUpdate(permSeqNum.incrementAndGet(), false); String authzObj = getAuthzObj(request.getAuthorizable()); update.addPrivilegeUpdate(authzObj).putToDelPrivileges( PermissionsUpdate.ALL_ROLES, PermissionsUpdate.ALL_ROLES); + permsUpdater.handleUpdateNotification(update); LOGGER.debug("Authz Perm preUpdate [" + update.getSeqNum() + ", " + authzObj + "].."); + return new DeltaTransactionBlock(update); } @Override http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java index 2ff715f..0b34e36 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/SentryPolicyStorePlugin.java @@ -20,6 +20,7 @@ package org.apache.sentry.provider.db; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.core.common.exception.SentryUserException; +import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleAddGroupsRequest; import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleDeleteGroupsRequest; @@ -28,7 +29,18 @@ import org.apache.sentry.provider.db.service.thrift.TAlterSentryRoleRevokePrivil import org.apache.sentry.provider.db.service.thrift.TDropPrivilegesRequest; import org.apache.sentry.provider.db.service.thrift.TDropSentryRoleRequest; import org.apache.sentry.provider.db.service.thrift.TRenamePrivilegesRequest; +import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; +import java.util.Map; + +/** + * Interface for processing delta changes of Sentry permission and generate corresponding + * update. The updates will be persisted into Sentry store afterwards along with the actual + * operation. + * + * TODO: SENTRY-1588: add user level privilege change support. e.g. onAlterSentryRoleDeleteUsers, + * TODO: onAlterSentryRoleDeleteUsers. + */ public interface SentryPolicyStorePlugin { @SuppressWarnings("serial") @@ -43,18 +55,19 @@ public interface SentryPolicyStorePlugin { void initialize(Configuration conf, SentryStore sentryStore) throws SentryPluginException; - void onAlterSentryRoleAddGroups(TAlterSentryRoleAddGroupsRequest tRequest) throws SentryPluginException; - - void onAlterSentryRoleDeleteGroups(TAlterSentryRoleDeleteGroupsRequest tRequest) throws SentryPluginException; + DeltaTransactionBlock onAlterSentryRoleAddGroups(TAlterSentryRoleAddGroupsRequest tRequest) throws SentryPluginException; - void onAlterSentryRoleGrantPrivilege(TAlterSentryRoleGrantPrivilegeRequest tRequest) throws SentryPluginException; + DeltaTransactionBlock onAlterSentryRoleDeleteGroups(TAlterSentryRoleDeleteGroupsRequest tRequest) throws SentryPluginException; - void onAlterSentryRoleRevokePrivilege(TAlterSentryRoleRevokePrivilegeRequest tRequest) throws SentryPluginException; + void onAlterSentryRoleGrantPrivilege(TAlterSentryRoleGrantPrivilegeRequest tRequest, + Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap) throws SentryPluginException; - void onDropSentryRole(TDropSentryRoleRequest tRequest) throws SentryPluginException; + void onAlterSentryRoleRevokePrivilege(TAlterSentryRoleRevokePrivilegeRequest tRequest, + Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap) throws SentryPluginException; - void onRenameSentryPrivilege(TRenamePrivilegesRequest request) throws SentryPluginException; + DeltaTransactionBlock onDropSentryRole(TDropSentryRoleRequest tRequest) throws SentryPluginException; - void onDropSentryPrivilege(TDropPrivilegesRequest request) throws SentryPluginException; + DeltaTransactionBlock onRenameSentryPrivilege(TRenamePrivilegesRequest request) throws SentryPluginException; + DeltaTransactionBlock onDropSentryPrivilege(TDropPrivilegesRequest request) throws SentryPluginException; } http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java index 8e2a6d5..1171a89 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/generic/service/persistent/DelegateSentryStore.java @@ -36,6 +36,7 @@ import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; import org.apache.sentry.provider.db.service.model.MSentryGMPrivilege; import org.apache.sentry.provider.db.service.model.MSentryGroup; import org.apache.sentry.provider.db.service.model.MSentryRole; +import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.persistent.TransactionBlock; import org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessor; @@ -92,7 +93,10 @@ public class DelegateSentryStore implements SentryStoreLayer { @Override public Object dropRole(final String component, final String role, final String requestor) throws Exception { - delegate.dropSentryRole(toTrimmedLower(role)); + // UpdateTransactionBlock is null for generic model. As we are + // only tracking permission update of Hive model for HDFS sync. + DeltaTransactionBlock deltaTransactionBlock = null; + delegate.dropSentryRole(toTrimmedLower(role), deltaTransactionBlock); return null; } @@ -104,15 +108,22 @@ public class DelegateSentryStore implements SentryStoreLayer { @Override public Object alterRoleAddGroups(String component, String role, Set<String> groups, String requestor) throws Exception { - delegate.alterSentryRoleAddGroups(requestor, role, toTSentryGroups(groups)); + // UpdateTransactionBlock is null for generic model. As we are + // only tracking permission update of Hive model for HDFS sync. + DeltaTransactionBlock deltaTransactionBlock = null; + delegate.alterSentryRoleAddGroups(requestor, role, toTSentryGroups(groups), + deltaTransactionBlock); return null; } @Override public Object alterRoleDeleteGroups(String component, String role, Set<String> groups, String requestor) throws Exception { - //called to old sentryStore - delegate.alterSentryRoleDeleteGroups(role, toTSentryGroups(groups)); + // Called to old sentryStore. UpdateTransactionBlock is null for generic model. + // As we are only tracking permission update of Hive model for HDFS sync. + DeltaTransactionBlock deltaTransactionBlock = null; + delegate.alterSentryRoleDeleteGroups(role, toTSentryGroups(groups), + deltaTransactionBlock); return null; } @@ -120,23 +131,21 @@ public class DelegateSentryStore implements SentryStoreLayer { public Object alterRoleGrantPrivilege(final String component, final String role, final PrivilegeObject privilege, final String grantorPrincipal) throws Exception { - delegate.getTransactionManager().executeTransactionWithRetry( - new TransactionBlock() { - public Object execute(PersistenceManager pm) throws Exception { - String trimmedRole = toTrimmedLower(role); - MSentryRole mRole = getRole(trimmedRole, pm); - if (mRole == null) { - throw new SentryNoSuchObjectException("Role: " + trimmedRole + " doesn't exist"); - } - /* - * check with grant option - */ - grantOptionCheck(privilege, grantorPrincipal, pm); + delegate.getTransactionManager().executeTransactionWithRetry(new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRole = toTrimmedLower(role); + MSentryRole mRole = getRole(trimmedRole, pm); + if (mRole == null) { + throw new SentryNoSuchObjectException("Role: " + trimmedRole + " doesn't exist"); + } - privilegeOperator.grantPrivilege(privilege, mRole, pm); - return null; - } - }); + // check with grant option + grantOptionCheck(privilege, grantorPrincipal, pm); + + privilegeOperator.grantPrivilege(privilege, mRole, pm); + return null; + } + }); return null; } @@ -144,23 +153,21 @@ public class DelegateSentryStore implements SentryStoreLayer { public Object alterRoleRevokePrivilege(final String component, final String role, final PrivilegeObject privilege, final String grantorPrincipal) throws Exception { - delegate.getTransactionManager().executeTransactionWithRetry( - new TransactionBlock() { - public Object execute(PersistenceManager pm) throws Exception { - String trimmedRole = toTrimmedLower(role); - MSentryRole mRole = getRole(trimmedRole, pm); - if (mRole == null) { - throw new SentryNoSuchObjectException("Role: " + trimmedRole + " doesn't exist"); - } - /* - * check with grant option - */ - grantOptionCheck(privilege, grantorPrincipal, pm); + delegate.getTransactionManager().executeTransactionWithRetry(new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRole = toTrimmedLower(role); + MSentryRole mRole = getRole(trimmedRole, pm); + if (mRole == null) { + throw new SentryNoSuchObjectException("Role: " + trimmedRole + " doesn't exist"); + } - privilegeOperator.revokePrivilege(privilege, mRole, pm); - return null; - } - }); + // check with grant option + grantOptionCheck(privilege, grantorPrincipal, pm); + + privilegeOperator.revokePrivilege(privilege, mRole, pm); + return null; + } + }); return null; } http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java index b88e7d1..0738589 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java @@ -63,24 +63,15 @@ public class MSentryPathChange { private String pathChange; private long createTimeMs; - public MSentryPathChange(long changeID, String pathChange, long createTime) { - this.changeID = changeID; + public MSentryPathChange(String pathChange) { this.pathChange = pathChange; - this.createTimeMs = createTime; - } - - public void setCreateTimeMs(long createTimeMs) { - this.createTimeMs = createTimeMs; + this.createTimeMs = System.currentTimeMillis(); } public long getCreateTimeMs() { return createTimeMs; } - public void setPathChange(String pathChange) { - this.pathChange = pathChange; - } - public String getPathChange() { return pathChange; } @@ -89,13 +80,10 @@ public class MSentryPathChange { return changeID; } - public void setChangeID(long changeID) { - this.changeID = changeID; - } - @Override public String toString() { - return "MSentryChange [changeID=" + changeID + " , pathChange= " + pathChange + ", createTime=" + createTimeMs + "]"; + return "MSentryChange [changeID=" + changeID + " , pathChange= " + pathChange + + ", createTime=" + createTimeMs + "]"; } @Override @@ -126,14 +114,14 @@ public class MSentryPathChange { return false; } - if (!pathChange.equals(other.pathChange)) { + if (createTimeMs != other.createTimeMs) { return false; } - if (createTimeMs != other.createTimeMs) { - return false; + if (pathChange == null) { + return other.pathChange == null; } - return true; + return pathChange.equals(other.pathChange); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java index 2ccace0..2100561 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPermChange.java @@ -61,24 +61,15 @@ public class MSentryPermChange { private String permChange; private long createTimeMs; - public MSentryPermChange(long changeID, String permChange, long createTimeMs) { - this.changeID = changeID; + public MSentryPermChange(String permChange) { this.permChange = permChange; - this.createTimeMs = createTimeMs; - } - - public void setCreateTimeMs(long createTimeMs) { - this.createTimeMs = createTimeMs; + this.createTimeMs = System.currentTimeMillis(); } public long getCreateTimeMs() { return createTimeMs; } - public void setPermChange(String permChange) { - this.permChange = permChange; - } - public String getPermChange() { return permChange; } @@ -87,13 +78,10 @@ public class MSentryPermChange { return changeID; } - public void setChangeID(long changeID) { - this.changeID = changeID; - } - @Override public String toString() { - return "MSentryPermChange [changeID=" + changeID + ", permChange= " + permChange + ", createTimeMs=" + createTimeMs + "]"; + return "MSentryPermChange [changeID=" + changeID + ", permChange= " + permChange + + ", createTimeMs=" + createTimeMs + "]"; } @Override @@ -128,10 +116,10 @@ public class MSentryPermChange { return false; } - if (!permChange.equals(other.permChange)) { - return false; + if (permChange == null) { + return other.permChange == null; } - return true; + return permChange.equals(other.permChange); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo index dc8fdbf..94ede1d 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo @@ -263,7 +263,7 @@ </class> <class name="MSentryPermChange" table="SENTRY_PERM_CHANGE" identity-type="application" detachable="true"> - <field name="changeID" primary-key="true"> + <field name="changeID" primary-key="true" value-strategy="increment"> <column name="CHANGE_ID" jdbc-type="BIGINT" allows-null="false"/> </field> <field name ="permChange"> @@ -275,7 +275,7 @@ </class> <class name="MSentryPathChange" table="SENTRY_PATH_CHANGE" identity-type="application" detachable="true"> - <field name="changeID" primary-key="true"> + <field name="changeID" primary-key="true" value-strategy="increment"> <column name="CHANGE_ID" jdbc-type="BIGINT" allows-null="false"/> </field> <field name ="pathChange"> http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java new file mode 100644 index 0000000..61145ab --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.provider.db.service.persistent; + +import org.apache.sentry.hdfs.PathsUpdate; +import org.apache.sentry.hdfs.PermissionsUpdate; +import org.apache.sentry.provider.db.service.model.MSentryPathChange; +import org.apache.sentry.provider.db.service.model.MSentryPermChange; +import static org.apache.sentry.hdfs.Updateable.Update; + +import javax.jdo.PersistenceManager; + +/** + * A subclass of {@link TransactionBlock} to manage the code should be executed for + * each delta update. The update could be {@link PathsUpdate} or {@link PermissionsUpdate}. + * Based on update type, the update would be persisted into corresponding update table, + * e.g {@link MSentryPathChange} {@link MSentryPermChange}. + * <p> + * Delta update should not have full image, hence update contains full image would not + * be executed. + */ +public class DeltaTransactionBlock implements TransactionBlock { + private final Update update; + + public DeltaTransactionBlock(Update update) { + this.update = update; + } + + @Override + public Object execute(PersistenceManager pm) throws Exception { + persistUpdate(pm, update); + return null; + } + + /** + * Persist the delta change into corresponding type based on its type. + * Atomic increasing primary key changeID by 1. Return without any + * operation if update is null or update is a fullImage. + * + * @param pm PersistenceManager + * @param update update + * @throws Exception + */ + private void persistUpdate(PersistenceManager pm, Update update) + throws Exception { + + // persistUpdate cannot handle full image update, instead + // it only handles delta updates. + if (update == null || update.hasFullImage()) { + return; + } + + // Persist the update into corresponding tables based on its type. + // changeID is the primary key in MSentryPXXXChange table. If same + // changeID is trying to be persisted twice, the transaction would + // fail. + if (update instanceof PermissionsUpdate) { + pm.makePersistent(new MSentryPermChange(update.JSONSerialize())); + } else if (update instanceof PathsUpdate) { + pm.makePersistent(new MSentryPathChange(update.JSONSerialize())); + } + } + + @Override + public int hashCode() { + return (update == null) ? 0 : update.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (this == obj) { + return true; + } + + if (getClass() != obj.getClass()) { + return false; + } + + DeltaTransactionBlock other = (DeltaTransactionBlock) obj; + if (update == null) { + return other.update == null; + } + return update.equals(other.update); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 0712e2c..1bf4e82 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -100,6 +100,12 @@ public class SentryStore { public static final String GRANT_OPTION = "grantOption"; public static final String ROLE_NAME = "roleName"; + // Initial change ID for permission/path change. Auto increment + // is starting from 1. + public static final long INIT_CHANGE_ID = 1L; + + private static final long EMPTY_CHANGE_ID = 0L; + // For counters, representation of the "unknown value" private static final long COUNT_VALUE_UNKNOWN = -1; @@ -426,43 +432,110 @@ public class SentryStore { } /** - * Grant privilege for a role + * Alter a given sentry role to grant a privilege. + * * @param grantorPrincipal User name - * @param roleName Role name - * @param privilege Privilege to grant + * @param roleName the given role name + * @param privilege the given privilege * @throws Exception */ - void alterSentryRoleGrantPrivilege(String grantorPrincipal, - String roleName, TSentryPrivilege privilege) throws Exception { - alterSentryRoleGrantPrivileges(grantorPrincipal, roleName, - Sets.newHashSet(privilege)); + void alterSentryRoleGrantPrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege privilege) throws Exception { + + tm.executeTransactionWithRetry( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRoleName = trimAndLower(roleName); + // first do grant check + grantOptionCheck(pm, grantorPrincipal, privilege); + + // Alter sentry Role and grant Privilege. + MSentryPrivilege mPrivilege = alterSentryRoleGrantPrivilegeCore( + pm, trimmedRoleName, privilege); + + if (mPrivilege != null) { + // update the privilege to be the one actually updated. + convertToTSentryPrivilege(mPrivilege, privilege); + } + return null; + } + }); } /** - * Grant multiple privileges + * Alter a given sentry role to grant a set of privileges. + * Internally calls alterSentryRoleGrantPrivilege. + * * @param grantorPrincipal User name * @param roleName Role name * @param privileges Set of privileges * @throws Exception */ - public void alterSentryRoleGrantPrivileges(final String grantorPrincipal, + void alterSentryRoleGrantPrivileges(final String grantorPrincipal, final String roleName, final Set<TSentryPrivilege> privileges) throws Exception { - tm.executeTransactionWithRetry( - new TransactionBlock() { - public Object execute(PersistenceManager pm) throws Exception { - String trimmedRoleName = trimAndLower(roleName); - for (TSentryPrivilege privilege : privileges) { - // first do grant check - grantOptionCheck(pm, grantorPrincipal, privilege); - MSentryPrivilege mPrivilege = alterSentryRoleGrantPrivilegeCore( - pm, trimmedRoleName, privilege); - if (mPrivilege != null) { - convertToTSentryPrivilege(mPrivilege, privilege); - } - } - return null; - } - }); + for (TSentryPrivilege privilege : privileges) { + alterSentryRoleGrantPrivilege(grantorPrincipal, roleName, privilege); + } + } + + /** + * Alter a given sentry role to grant a privilege, as well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param privilege the given privilege + * @param deltaTransactionBlock the corresponding permission delta update transaction block + * @throws Exception + * + */ + void alterSentryRoleGrantPrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege privilege, + final DeltaTransactionBlock deltaTransactionBlock) throws Exception { + + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRoleName = trimAndLower(roleName); + // first do grant check + grantOptionCheck(pm, grantorPrincipal, privilege); + + // Alter sentry Role and grant Privilege. + MSentryPrivilege mPrivilege = alterSentryRoleGrantPrivilegeCore(pm, + trimmedRoleName, privilege); + + if (mPrivilege != null) { + // update the privilege to be the one actually updated. + convertToTSentryPrivilege(mPrivilege, privilege); + } + return null; + } + }); + } + + /** + * Alter a given sentry role to grant a set of privileges, as well as persist the + * corresponding permission change to MSentryPermChange table in a single transaction. + * Internally calls alterSentryRoleGrantPrivilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param privileges a Set of privileges + * @param privilegesUpdateMap the corresponding <privilege, DeltaTransactionBlock> map + * @throws Exception + * + */ + public void alterSentryRoleGrantPrivileges(final String grantorPrincipal, + final String roleName, final Set<TSentryPrivilege> privileges, + final Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap) throws Exception { + + for (TSentryPrivilege privilege : privileges) { + DeltaTransactionBlock deltaTransactionBlock = null; + if (privilegesUpdateMap != null) { + deltaTransactionBlock = privilegesUpdateMap.get(privilege); + } + alterSentryRoleGrantPrivilege(grantorPrincipal, roleName, privilege, + deltaTransactionBlock); + } } private MSentryPrivilege alterSentryRoleGrantPrivilegeCore(PersistenceManager pm, @@ -522,27 +595,99 @@ public class SentryStore { return mPrivilege; } - void alterSentryRoleRevokePrivilege(String grantorPrincipal, - String roleName, TSentryPrivilege tPrivilege) throws Exception { - alterSentryRoleRevokePrivileges(grantorPrincipal, roleName, - Sets.newHashSet(tPrivilege)); + /** + * Alter a given sentry role to revoke a privilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivilege the given privilege + * @throws Exception + * + */ + void alterSentryRoleRevokePrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege tPrivilege) throws Exception { + + tm.executeTransactionWithRetry( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRoleName = safeTrimLower(roleName); + // first do revoke check + grantOptionCheck(pm, grantorPrincipal, tPrivilege); + + alterSentryRoleRevokePrivilegeCore(pm, trimmedRoleName, tPrivilege); + return null; + } + }); } - public void alterSentryRoleRevokePrivileges(final String grantorPrincipal, + /** + * Alter a given sentry role to revoke a set of privileges. + * Internally calls alterSentryRoleRevokePrivilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivileges a Set of privileges + * @throws Exception + * + */ + void alterSentryRoleRevokePrivileges(final String grantorPrincipal, final String roleName, final Set<TSentryPrivilege> tPrivileges) throws Exception { - tm.executeTransactionWithRetry( - new TransactionBlock() { - public Object execute(PersistenceManager pm) throws Exception { - String trimmedRoleName = safeTrimLower(roleName); - for (TSentryPrivilege tPrivilege : tPrivileges) { - // first do revoke check - grantOptionCheck(pm, grantorPrincipal, tPrivilege); + for (TSentryPrivilege tPrivilege : tPrivileges) { + alterSentryRoleRevokePrivilege(grantorPrincipal, roleName, tPrivilege); + } + } - alterSentryRoleRevokePrivilegeCore(pm, trimmedRoleName, tPrivilege); - } - return null; - } - }); + /** + * Alter a given sentry role to revoke a privilege, as well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivilege the given privilege + * @param deltaTransactionBlock the corresponding permission delta update transaction block + * @throws Exception + * + */ + void alterSentryRoleRevokePrivilege(final String grantorPrincipal, + final String roleName, final TSentryPrivilege tPrivilege, + final DeltaTransactionBlock deltaTransactionBlock) throws Exception { + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRoleName = safeTrimLower(roleName); + // first do revoke check + grantOptionCheck(pm, grantorPrincipal, tPrivilege); + + alterSentryRoleRevokePrivilegeCore(pm, trimmedRoleName, tPrivilege); + return null; + } + }); + } + + /** + * Alter a given sentry role to revoke a set of privileges, as well as persist the + * corresponding permission change to MSentryPermChange table in a single transaction. + * Internally calls alterSentryRoleRevokePrivilege. + * + * @param grantorPrincipal User name + * @param roleName the given role name + * @param tPrivileges a Set of privileges + * @param privilegesUpdateMap the corresponding <privilege, DeltaTransactionBlock> map + * @throws Exception + * + */ + public void alterSentryRoleRevokePrivileges(final String grantorPrincipal, + final String roleName, final Set<TSentryPrivilege> tPrivileges, + final Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap) + throws Exception { + + for (TSentryPrivilege tPrivilege : tPrivileges) { + DeltaTransactionBlock deltaTransactionBlock = null; + if (privilegesUpdateMap != null) { + deltaTransactionBlock = privilegesUpdateMap.get(tPrivilege); + } + alterSentryRoleRevokePrivilege(grantorPrincipal, roleName, tPrivilege, + deltaTransactionBlock); + } } private void alterSentryRoleRevokePrivilegeCore(PersistenceManager pm, @@ -795,6 +940,12 @@ public class SentryStore { return (MSentryPrivilege)query.executeWithMap(paramBuilder.getArguments()); } + /** + * Drop a given sentry role. + * + * @param roleName the given role name + * @throws Exception + */ public void dropSentryRole(final String roleName) throws Exception { tm.executeTransactionWithRetry( new TransactionBlock() { @@ -805,6 +956,25 @@ public class SentryStore { }); } + /** + * Drop a given sentry role. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param roleName the given role name + * @param deltaTransactionBlock the corresponding permission delta update transaction block + * @throws Exception + */ + public void dropSentryRole(final String roleName, + final DeltaTransactionBlock deltaTransactionBlock) throws Exception { + + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + dropSentryRoleCore(pm, roleName); + return null; + } + }); + } + private void dropSentryRoleCore(PersistenceManager pm, String roleName) throws SentryNoSuchObjectException { String lRoleName = trimAndLower(roleName); @@ -820,7 +990,15 @@ public class SentryStore { pm.deletePersistent(sentryRole); } - public void alterSentryRoleAddGroups(final String grantorPrincipal, + /** + * Assign a given role to a set of groups. + * + * @param grantorPrincipal grantorPrincipal currently is not used. + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @throws Exception + */ + void alterSentryRoleAddGroups(final String grantorPrincipal, final String roleName, final Set<TSentryGroup> groupNames) throws Exception { tm.executeTransactionWithRetry( new TransactionBlock() { @@ -831,13 +1009,39 @@ public class SentryStore { }); } + /** + * Assign a given role to a set of groups. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param grantorPrincipal grantorPrincipal currently is not used. + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @param deltaTransactionBlock the corresponding permission delta update transaction block + * @throws Exception + */ + public void alterSentryRoleAddGroups(final String grantorPrincipal, + final String roleName, final Set<TSentryGroup> groupNames, + final DeltaTransactionBlock deltaTransactionBlock) throws Exception { + + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + alterSentryRoleAddGroupsCore(pm, roleName, groupNames); + return null; + } + }); + } + private void alterSentryRoleAddGroupsCore(PersistenceManager pm, String roleName, Set<TSentryGroup> groupNames) throws SentryNoSuchObjectException { + + // All role names are stored in lowercase. String lRoleName = trimAndLower(roleName); MSentryRole role = getRole(pm, lRoleName); if (role == null) { throw noSuchRole(lRoleName); } + + // Add the group to the specified role if it does not belong to the role yet. Query query = pm.newQuery(MSentryGroup.class); query.setFilter("this.groupName == :groupName"); query.setUnique(true); @@ -917,7 +1121,14 @@ public class SentryStore { }); } - public void alterSentryRoleDeleteGroups(final String roleName, + /** + * Revoke a given role to a set of groups. + * + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @throws Exception + */ + void alterSentryRoleDeleteGroups(final String roleName, final Set<TSentryGroup> groupNames) throws Exception { tm.executeTransactionWithRetry( new TransactionBlock() { @@ -945,6 +1156,45 @@ public class SentryStore { }); } + /** + * Revoke a given role to a set of groups. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param roleName the role to be assigned to the groups. + * @param groupNames the list of groups to be added to the role, + * @param deltaTransactionBlock the corresponding permission delta update transaction block + * @throws Exception + */ + public void alterSentryRoleDeleteGroups(final String roleName, + final Set<TSentryGroup> groupNames, final DeltaTransactionBlock deltaTransactionBlock) + throws Exception { + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + String trimmedRoleName = trimAndLower(roleName); + MSentryRole role = getRole(pm, trimmedRoleName); + if (role == null) { + throw noSuchRole(trimmedRoleName); + } + + // Remove the group from the specified role if it belongs to the role. + Query query = pm.newQuery(MSentryGroup.class); + query.setFilter("this.groupName == :groupName"); + query.setUnique(true); + List<MSentryGroup> groups = Lists.newArrayList(); + for (TSentryGroup tGroup : groupNames) { + String groupName = tGroup.getGroupName().trim(); + MSentryGroup group = (MSentryGroup) query.execute(groupName); + if (group != null) { + group.removeRole(role); + groups.add(group); + } + } + pm.makePersistentAll(groups); + return null; + } + }); + } + @VisibleForTesting MSentryRole getMSentryRoleByName(final String roleName) throws Exception { return tm.executeTransaction( @@ -1545,14 +1795,19 @@ public class SentryStore { } /** - * Drop given privilege from all roles + * Drop the given privilege from all roles. + * + * @param tAuthorizable the given authorizable object. + * @throws Exception */ - public void dropPrivilege(final TSentryAuthorizable tAuthorizable) throws Exception { + void dropPrivilege(final TSentryAuthorizable tAuthorizable) throws Exception { tm.executeTransactionWithRetry( new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { + // Drop the give privilege for all possible actions from all roles. TSentryPrivilege tPrivilege = toSentryPrivilege(tAuthorizable); + try { if (isMultiActionsSupported(tPrivilege)) { for (String privilegeAction : ALL_ACTIONS) { @@ -1572,20 +1827,58 @@ public class SentryStore { } /** - * Rename given privilege from all roles drop the old privilege and create the new one - * @param tAuthorizable - * @param newTAuthorizable + * Drop the given privilege from all roles. As well as persist the corresponding + * permission change to MSentryPermChange table in a single transaction. + * + * @param tAuthorizable the given authorizable object. + * @param deltaTransactionBlock the corresponding permission delta update transaction block. + * @throws Exception + */ + public void dropPrivilege(final TSentryAuthorizable tAuthorizable, + final DeltaTransactionBlock deltaTransactionBlock) throws Exception { + + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + + // Drop the give privilege for all possible actions from all roles. + TSentryPrivilege tPrivilege = toSentryPrivilege(tAuthorizable); + + try { + if (isMultiActionsSupported(tPrivilege)) { + for (String privilegeAction : ALL_ACTIONS) { + tPrivilege.setAction(privilegeAction); + dropPrivilegeForAllRoles(pm, new TSentryPrivilege(tPrivilege)); + } + } else { + dropPrivilegeForAllRoles(pm, new TSentryPrivilege(tPrivilege)); + } + } catch (JDODataStoreException e) { + throw new SentryInvalidInputException("Failed to get privileges: " + + e.getMessage()); + } + return null; + } + }); + } + + /** + * Rename the privilege for all roles. Drop the old privilege name and create the new one. + * + * @param oldTAuthorizable the old authorizable name needs to be renamed. + * @param newTAuthorizable the new authorizable name * @throws SentryNoSuchObjectException * @throws SentryInvalidInputException */ - public void renamePrivilege(final TSentryAuthorizable tAuthorizable, + void renamePrivilege(final TSentryAuthorizable oldTAuthorizable, final TSentryAuthorizable newTAuthorizable) throws Exception { tm.executeTransactionWithRetry( new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { - TSentryPrivilege tPrivilege = toSentryPrivilege(tAuthorizable); + // Drop the give privilege for all possible actions from all roles. + TSentryPrivilege tPrivilege = toSentryPrivilege(oldTAuthorizable); TSentryPrivilege newPrivilege = toSentryPrivilege(newTAuthorizable); + try { // In case of tables or DBs, check all actions if (isMultiActionsSupported(tPrivilege)) { @@ -1606,6 +1899,48 @@ public class SentryStore { }); } + /** + * Rename the privilege for all roles. Drop the old privilege name and create the new one. + * As well as persist the corresponding permission change to MSentryPermChange table in a + * single transaction. + * + * @param oldTAuthorizable the old authorizable name needs to be renamed. + * @param newTAuthorizable the new authorizable name + * @param deltaTransactionBlock the corresponding permission delta update transaction block + * @throws SentryNoSuchObjectException + * @throws SentryInvalidInputException + */ + public void renamePrivilege(final TSentryAuthorizable oldTAuthorizable, + final TSentryAuthorizable newTAuthorizable, final DeltaTransactionBlock deltaTransactionBlock) + throws Exception { + + execute(deltaTransactionBlock, new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + + // Drop the give privilege for all possible actions from all roles. + TSentryPrivilege tPrivilege = toSentryPrivilege(oldTAuthorizable); + TSentryPrivilege newPrivilege = toSentryPrivilege(newTAuthorizable); + + try { + // In case of tables or DBs, check all actions + if (isMultiActionsSupported(tPrivilege)) { + for (String privilegeAction : ALL_ACTIONS) { + tPrivilege.setAction(privilegeAction); + newPrivilege.setAction(privilegeAction); + renamePrivilegeForAllRoles(pm, tPrivilege, newPrivilege); + } + } else { + renamePrivilegeForAllRoles(pm, tPrivilege, newPrivilege); + } + } catch (JDODataStoreException e) { + throw new SentryInvalidInputException("Failed to get privileges: " + + e.getMessage()); + } + return null; + } + }); + } + // Currently INSERT/SELECT/ALL are supported for Table and DB level privileges private boolean isMultiActionsSupported(TSentryPrivilege tPrivilege) { return tPrivilege.getDbName() != null; @@ -2620,6 +2955,15 @@ public class SentryStore { } /** + * Return exception for nonexistent update + * @param changeID change ID + * @return SentryNoSuchObjectException with appropriate message + */ + private SentryNoSuchObjectException noSuchUpdate(final long changeID) { + return new SentryNoSuchObjectException("nonexistent update + " + changeID); + } + + /** * Add common filter for set of roles * @param query Query used for search * @param paramBuilder paramBuilder for parameters @@ -2926,4 +3270,107 @@ public class SentryStore { return this; } } + + /** + * Get the last processed perm change ID. + * + * @param pm the PersistenceManager + * @return the last processed perm changedID + */ + private long getLastProcessedPermChangeIDCore(PersistenceManager pm) { + Query query = pm.newQuery(MSentryPermChange.class); + query.setResult("max(this.changeID)"); + Long changeID = (Long) query.execute(); + if (changeID == null) { + return EMPTY_CHANGE_ID; + } else { + return changeID; + } + } + + /** + * Get the MSentryPermChange object by ChangeID. Internally invoke + * getLastProcessedPermChangeIDCore(). + * + * @return MSentryPermChange + */ + @VisibleForTesting + long getLastProcessedPermChangeID() throws Exception { + return tm.executeTransaction( + new TransactionBlock<Long>() { + public Long execute(PersistenceManager pm) throws Exception { + return getLastProcessedPermChangeIDCore(pm); + } + }); + } + + /** + * Get the MSentryPermChange object by ChangeID. + * + * @param changeID the given changeID. + * @return MSentryPermChange + */ + public MSentryPermChange getMSentryPermChangeByID(final long changeID) throws Exception { + return (MSentryPermChange) tm.executeTransaction( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + Query query = pm.newQuery(MSentryPermChange.class); + query.setFilter("this.changeID == t"); + query.declareParameters("long t"); + List<MSentryPermChange> permChanges = (List<MSentryPermChange>)query.execute(changeID); + if (permChanges == null) { + noSuchUpdate(changeID); + } else if (permChanges.size() > 1) { + throw new Exception("Each change ID should only corresponds to one perm change!"); + } + + return permChanges.get(0); + } + }); + } + + /** + * Get the MSentryPathChange object by ChangeID. + */ + public MSentryPathChange getMSentryPathChangeByID(final long changeID) throws Exception { + return (MSentryPathChange) tm.executeTransaction( + new TransactionBlock() { + public Object execute(PersistenceManager pm) throws Exception { + Query query = pm.newQuery(MSentryPathChange.class); + query.setFilter("this.changeID == t"); + query.declareParameters("long t"); + List<MSentryPathChange> pathChanges = (List<MSentryPathChange>)query.execute(changeID); + if (pathChanges == null) { + noSuchUpdate(changeID); + } else if (pathChanges.size() > 1) { + throw new Exception("Each change ID should only corresponds to one path change!"); + } + + return pathChanges.get(0); + } + }); + } + + /** + * Execute Perm/Path UpdateTransaction and corresponding actual + * action transaction, e.g dropSentryRole, in a single transaction. + * The order of the transaction does not matter because there is no + * any return value. + * <p> + * Failure in any TransactionBlock would cause the whole transaction + * to fail. + * + * @param deltaTransactionBlock + * @param transactionBlock + * @throws Exception + */ + private void execute(DeltaTransactionBlock deltaTransactionBlock, + TransactionBlock<Object> transactionBlock) throws Exception { + List<TransactionBlock<Object>> tbs = Lists.newArrayList(); + if (deltaTransactionBlock != null) { + tbs.add(deltaTransactionBlock); + } + tbs.add(transactionBlock); + tm.executeTransactionBlocksWithRetry(tbs); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/TransactionManager.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/TransactionManager.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/TransactionManager.java index 6428a0c..40fd58b 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/TransactionManager.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/TransactionManager.java @@ -34,12 +34,14 @@ import javax.jdo.Transaction; import org.apache.sentry.provider.db.service.thrift.SentryMetrics; +import java.util.List; /** * TransactionManager is used for executing the database transaction, it supports * the transaction with retry mechanism for the unexpected exceptions, * except <em>SentryUserExceptions</em>, eg, <em>SentryNoSuchObjectException</em>, - * <em>SentryAlreadyExistsException</em> etc. <p> + * <em>SentryAlreadyExistsException</em> etc. For <em>SentryUserExceptions</em>, + * will simply throw the exception without retry<p> * * The purpose of the class is to separate all transaction housekeeping (opening * transaction, rolling back failed transactions) from the actual transaction @@ -99,7 +101,8 @@ public class TransactionManager { * Execute some code as a single transaction, the code in tb.execute() * should not start new transaction or manipulate transactions with the * PersistenceManager. - * @param tb transaction block with code to execute + * + * @param tb transaction block with code to be executed * @return Object with the result of tb.execute() */ public <T> T executeTransaction(TransactionBlock<T> tb) throws Exception { @@ -129,7 +132,46 @@ public class TransactionManager { } /** - * Execute some code as a single transaction with retry mechanism + * Execute a list of TransactionBlock code as a single transaction. + * The code in tb.execute() should not start new transaction or + * manipulate transactions with the PersistenceManager. It returns + * the result of the last transaction block execution. + * + * @param tbs transaction blocks with code to be executed + * @return the result of the last result of tb.execute() + */ + public <T> T executeTransaction(Iterable<TransactionBlock<T>> tbs) throws Exception { + final Timer.Context context = transactionTimer.time(); + try (PersistenceManager pm = pmf.getPersistenceManager()) { + Transaction transaction = pm.currentTransaction(); + transaction.begin(); + try { + T result = null; + for (TransactionBlock<T> tb : tbs) { + result = tb.execute(pm); + } + transaction.commit(); + return result; + } catch (Exception e) { + // Count total failed transactions + failedTransactionsCount.inc(); + // Count specific exceptions + SentryMetrics.getInstance().getCounter(name(TransactionManager.class, + "exception", e.getClass().getSimpleName())).inc(); + // Re-throw the exception + throw e; + } finally { + context.stop(); + if (transaction.isActive()) { + transaction.rollback(); + } + } + } + } + + /** + * Execute some code as a single transaction with retry mechanism. + * * @param tb transaction block with code to execute * @return Object with the result of tb.execute() */ @@ -139,20 +181,55 @@ public class TransactionManager { while (retryNum < transactionRetryMax) { try { return executeTransaction(tb); - } catch (Exception e) { + } catch (SentryUserException e) { // throw the sentry exception without retry - if (e instanceof SentryUserException) { - throw e; + throw e; + } catch (Exception e) { + retryNum++; + if (retryNum >= transactionRetryMax) { + String message = "The transaction has reached max retry numbe, r" + + e.getMessage(); + LOGGER.error(message, e); + throw new Exception(message, e); } + retryCount.inc(); + LOGGER.warn("Exception during transaction execution, retrying " + + retryNum + "times. The max retry num is: " + transactionRetryMax, e); + Thread.sleep(retryWaitTimeMills); + } + } + return null; + } + + /** + * Execute a list of TransactionBlock code as a single transaction. + * If any of the TransactionBlock fail, all the TransactionBlocks would + * retry. It returns the result of the last transaction block + * execution. + * + * @param tbs a list of transaction blocks with code to be executed. + * @return the result of the last transaction block execution. + */ + public <T> T executeTransactionBlocksWithRetry(Iterable<TransactionBlock<T>> tbs) + throws Exception { + int retryNum = 0; + while (retryNum < transactionRetryMax) { + try { + return executeTransaction(tbs); + } catch (SentryUserException e) { + // throw the sentry exception without retry + throw e; + } catch (Exception e) { retryNum++; if (retryNum >= transactionRetryMax) { - String message = "The transaction has reached max retry number, will not retry again."; + String message = "The transaction has reached max retry number, " + + e.getMessage(); LOGGER.error(message, e); throw new Exception(message, e); } retryCount.inc(); - LOGGER.warn("Exception is thrown, retry the transaction, current retry num is:" - + retryNum + ", the max retry num is: " + transactionRetryMax, e); + LOGGER.warn("Exception during transaction execution, retrying " + + retryNum + "times. The max retry num is: " + transactionRetryMax, e); Thread.sleep(retryWaitTimeMills); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java index 5121740..d38a2dc 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyStoreProcessor.java @@ -20,6 +20,7 @@ package org.apache.sentry.provider.db.service.thrift; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -46,6 +47,7 @@ import org.apache.sentry.provider.db.log.entity.JsonLogEntity; import org.apache.sentry.provider.db.log.entity.JsonLogEntityFactory; import org.apache.sentry.provider.db.log.util.Constants; import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; import org.apache.sentry.provider.db.service.thrift.PolicyStoreConstants.PolicyStoreServerConfig; import org.apache.sentry.service.thrift.SentryServiceUtil; import org.apache.sentry.service.thrift.ServiceConstants; @@ -74,8 +76,6 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { public static final String SENTRY_POLICY_SERVICE_NAME = "SentryPolicyService"; - public static volatile SentryPolicyStoreProcessor instance; - private final String name; private final Configuration conf; private final SentryStore sentryStore; @@ -109,9 +109,6 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { plugin.initialize(conf, sentryStore); sentryPlugins.add(plugin); } - if (instance == null) { - instance = this; - } initMetrics(); } @@ -244,8 +241,17 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { if (request.isSetPrivilege()) { request.setPrivileges(Sets.newHashSet(request.getPrivilege())); } + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap = new HashMap<>(); + for (SentryPolicyStorePlugin plugin : sentryPlugins) { + plugin.onAlterSentryRoleGrantPrivilege(request, privilegesUpdateMap); + } + sentryStore.alterSentryRoleGrantPrivileges(request.getRequestorUserName(), - request.getRoleName(), request.getPrivileges()); + request.getRoleName(), request.getPrivileges(), privilegesUpdateMap); response.setStatus(Status.OK()); response.setPrivileges(request.getPrivileges()); // Maintain compatibility for old API: Set privilege field to response @@ -254,9 +260,6 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { } notificationHandlerInvoker.alter_sentry_role_grant_privilege(request, response); - for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onAlterSentryRoleGrantPrivilege(request); - } } catch (SentryNoSuchObjectException e) { String msg = "Role: " + request.getRoleName() + " doesn't exist"; LOGGER.error(msg, e); @@ -308,14 +311,20 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { if (request.isSetPrivilege()) { request.setPrivileges(Sets.newHashSet(request.getPrivilege())); } + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + Map<TSentryPrivilege, DeltaTransactionBlock> privilegesUpdateMap = new HashMap<> (); + for (SentryPolicyStorePlugin plugin : sentryPlugins) { + plugin.onAlterSentryRoleRevokePrivilege(request, privilegesUpdateMap); + } + sentryStore.alterSentryRoleRevokePrivileges(request.getRequestorUserName(), - request.getRoleName(), request.getPrivileges()); + request.getRoleName(), request.getPrivileges(), privilegesUpdateMap); response.setStatus(Status.OK()); notificationHandlerInvoker.alter_sentry_role_revoke_privilege(request, response); - for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onAlterSentryRoleRevokePrivilege(request); - } } catch (SentryNoSuchObjectException e) { StringBuilder msg = new StringBuilder(); if (request.getPrivileges().size() > 0) { @@ -378,12 +387,18 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { validateClientVersion(request.getProtocol_version()); authorize(request.getRequestorUserName(), getRequestorGroups(request.getRequestorUserName())); - sentryStore.dropSentryRole(request.getRoleName()); - response.setStatus(Status.OK()); - notificationHandlerInvoker.drop_sentry_role(request, response); + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + DeltaTransactionBlock deltaTransactionBlock = null; for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onDropSentryRole(request); + deltaTransactionBlock = plugin.onDropSentryRole(request); } + + sentryStore.dropSentryRole(request.getRoleName(), deltaTransactionBlock); + response.setStatus(Status.OK()); + notificationHandlerInvoker.drop_sentry_role(request, response); } catch (SentryNoSuchObjectException e) { String msg = "Role :" + request + " doesn't exist"; LOGGER.error(msg, e); @@ -422,15 +437,21 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { validateClientVersion(request.getProtocol_version()); authorize(request.getRequestorUserName(), getRequestorGroups(request.getRequestorUserName())); + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + DeltaTransactionBlock deltaTransactionBlock = null; + for (SentryPolicyStorePlugin plugin : sentryPlugins) { + deltaTransactionBlock = plugin.onAlterSentryRoleAddGroups(request); + } + sentryStore.alterSentryRoleAddGroups( request.getRequestorUserName(), request.getRoleName(), - request.getGroups()); + request.getGroups(), deltaTransactionBlock); response.setStatus(Status.OK()); notificationHandlerInvoker.alter_sentry_role_add_groups(request, - response); - for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onAlterSentryRoleAddGroups(request); - } + response); } catch (SentryNoSuchObjectException e) { String msg = "Role: " + request + " doesn't exist"; LOGGER.error(msg, e); @@ -550,14 +571,20 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { validateClientVersion(request.getProtocol_version()); authorize(request.getRequestorUserName(), getRequestorGroups(request.getRequestorUserName())); + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + DeltaTransactionBlock deltaTransactionBlock = null; + for (SentryPolicyStorePlugin plugin : sentryPlugins) { + deltaTransactionBlock = plugin.onAlterSentryRoleDeleteGroups(request); + } + sentryStore.alterSentryRoleDeleteGroups(request.getRoleName(), - request.getGroups()); + request.getGroups(), deltaTransactionBlock); response.setStatus(Status.OK()); notificationHandlerInvoker.alter_sentry_role_delete_groups(request, - response); - for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onAlterSentryRoleDeleteGroups(request); - } + response); } catch (SentryNoSuchObjectException e) { String msg = "Role: " + request + " does not exist."; LOGGER.error(msg, e); @@ -829,10 +856,16 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { try { validateClientVersion(request.getProtocol_version()); authorize(request.getRequestorUserName(), adminGroups); - sentryStore.dropPrivilege(request.getAuthorizable()); + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + DeltaTransactionBlock deltaTransactionBlock = null; for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onDropSentryPrivilege(request); + deltaTransactionBlock = plugin.onDropSentryPrivilege(request); } + + sentryStore.dropPrivilege(request.getAuthorizable(), deltaTransactionBlock); response.setStatus(Status.OK()); } catch (SentryAccessDeniedException e) { LOGGER.error(e.getMessage(), e); @@ -859,11 +892,17 @@ public class SentryPolicyStoreProcessor implements SentryPolicyService.Iface { try { validateClientVersion(request.getProtocol_version()); authorize(request.getRequestorUserName(), adminGroups); - sentryStore.renamePrivilege(request.getOldAuthorizable(), - request.getNewAuthorizable()); + + // TODO: now only has SentryPlugin. Once add more SentryPolicyStorePlugins, + // TODO: need to differentiate the updates for different Plugins. + Preconditions.checkState(sentryPlugins.size() <= 1); + DeltaTransactionBlock deltaTransactionBlock = null; for (SentryPolicyStorePlugin plugin : sentryPlugins) { - plugin.onRenameSentryPrivilege(request); + deltaTransactionBlock = plugin.onRenameSentryPrivilege(request); } + + sentryStore.renamePrivilege(request.getOldAuthorizable(), + request.getNewAuthorizable(), deltaTransactionBlock); response.setStatus(Status.OK()); } catch (SentryAccessDeniedException e) { LOGGER.error(e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/sentry/blob/ee2d3f7a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index 749c2ce..59646b6 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -30,8 +30,12 @@ import org.apache.hadoop.security.SaslRpcServer; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.core.common.exception.*; +import org.apache.sentry.hdfs.PermissionsUpdate; import org.apache.sentry.hdfs.UpdateableAuthzPaths; import org.apache.sentry.hdfs.FullUpdateInitializer; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.sentry.provider.db.SentryPolicyStorePlugin; +import org.apache.sentry.provider.db.service.persistent.DeltaTransactionBlock; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; import org.apache.thrift.TException; @@ -439,14 +443,16 @@ public class HMSFollower implements Runnable { private void dropSentryDbPrivileges(String dbName) throws Exception { TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); authorizable.setDb(dbName); - sentryStore.dropPrivilege(authorizable); + sentryStore.dropPrivilege(authorizable, onDropSentryPrivilege(authorizable)); } + private void dropSentryTablePrivileges(String dbName, String tableName) throws Exception { TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); authorizable.setDb(dbName); authorizable.setTable(tableName); - sentryStore.dropPrivilege(authorizable); + sentryStore.dropPrivilege(authorizable, onDropSentryPrivilege(authorizable)); } + private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws Exception { TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance); @@ -455,6 +461,43 @@ public class HMSFollower implements Runnable { TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); newAuthorizable.setDb(newDbName); newAuthorizable.setTable(newTableName); - sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable); + DeltaTransactionBlock deltaTransactionBlock = + onRenameSentryPrivilege(oldAuthorizable, newAuthorizable); + sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable, deltaTransactionBlock); + } + + @VisibleForTesting + static DeltaTransactionBlock onDropSentryPrivilege(TSentryAuthorizable authorizable) { + PermissionsUpdate update = new PermissionsUpdate(SentryStore.INIT_CHANGE_ID, false); + String authzObj = getAuthzObj(authorizable); + update.addPrivilegeUpdate(authzObj).putToDelPrivileges(PermissionsUpdate.ALL_ROLES, PermissionsUpdate.ALL_ROLES); + return new DeltaTransactionBlock(update); + } + + @VisibleForTesting + static DeltaTransactionBlock onRenameSentryPrivilege(TSentryAuthorizable oldAuthorizable, + TSentryAuthorizable newAuthorizable) + throws SentryPolicyStorePlugin.SentryPluginException { + String oldAuthz = getAuthzObj(oldAuthorizable); + String newAuthz = getAuthzObj(newAuthorizable); + PermissionsUpdate update = new PermissionsUpdate(SentryStore.INIT_CHANGE_ID, false); + TPrivilegeChanges privUpdate = update.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS); + privUpdate.putToAddPrivileges(newAuthz, newAuthz); + privUpdate.putToDelPrivileges(oldAuthz, oldAuthz); + return new DeltaTransactionBlock(update); + } + + private static String getAuthzObj(TSentryAuthorizable authzble) { + String authzObj = null; + if (!SentryStore.isNULL(authzble.getDb())) { + String dbName = authzble.getDb(); + String tblName = authzble.getTable(); + if (SentryStore.isNULL(tblName)) { + authzObj = dbName; + } else { + authzObj = dbName + "." + tblName; + } + } + return authzObj == null ? null : authzObj.toLowerCase(); } }
