Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign b98b587e9 -> 2911c532b
http://git-wip-us.apache.org/repos/asf/sentry/blob/2911c532/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index 1f05ba9..ecd1175 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -32,8 +32,11 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.core.common.exception.*; +import org.apache.sentry.hdfs.PermissionsUpdate; import org.apache.sentry.hdfs.UpdateableAuthzPaths; import org.apache.sentry.hdfs.FullUpdateInitializer; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.sentry.provider.db.SentryPolicyStorePlugin; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; import org.apache.thrift.TException; @@ -51,6 +54,7 @@ import java.util.List; import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE; import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE; +import static org.apache.sentry.hdfs.Updateable.Update; /** * HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry. @@ -437,14 +441,16 @@ public class HMSFollower implements Runnable { private void dropSentryDbPrivileges(String dbName) throws Exception { TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); authorizable.setDb(dbName); - sentryStore.dropPrivilege(authorizable); + sentryStore.dropPrivilege(authorizable, onDropSentryPrivilege(authorizable)); } + private void dropSentryTablePrivileges(String dbName, String tableName) throws Exception { TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); authorizable.setDb(dbName); authorizable.setTable(tableName); - sentryStore.dropPrivilege(authorizable); + sentryStore.dropPrivilege(authorizable, onDropSentryPrivilege(authorizable)); } + private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws Exception { TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance); @@ -453,6 +459,43 @@ public class HMSFollower implements Runnable { TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); newAuthorizable.setDb(newDbName); newAuthorizable.setTable(newTableName); - sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable); + Update update = + onRenameSentryPrivilege(oldAuthorizable, newAuthorizable); + sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable, update); + } + + @VisibleForTesting + static Update onDropSentryPrivilege(TSentryAuthorizable authorizable) { + PermissionsUpdate update = new PermissionsUpdate(SentryStore.INIT_CHANGE_ID, false); + String authzObj = getAuthzObj(authorizable); + update.addPrivilegeUpdate(authzObj).putToDelPrivileges(PermissionsUpdate.ALL_ROLES, PermissionsUpdate.ALL_ROLES); + return update; + } + + @VisibleForTesting + static Update onRenameSentryPrivilege(TSentryAuthorizable oldAuthorizable, + TSentryAuthorizable newAuthorizable) + throws SentryPolicyStorePlugin.SentryPluginException { + String oldAuthz = getAuthzObj(oldAuthorizable); + String newAuthz = getAuthzObj(newAuthorizable); + PermissionsUpdate update = new PermissionsUpdate(SentryStore.INIT_CHANGE_ID, false); + TPrivilegeChanges privUpdate = update.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS); + privUpdate.putToAddPrivileges(newAuthz, newAuthz); + privUpdate.putToDelPrivileges(oldAuthz, oldAuthz); + return update; + } + + public static String getAuthzObj(TSentryAuthorizable authzble) { + String authzObj = null; + if (!SentryStore.isNULL(authzble.getDb())) { + String dbName = authzble.getDb(); + String tblName = authzble.getTable(); + if (SentryStore.isNULL(tblName)) { + authzObj = dbName; + } else { + authzObj = dbName + "." + tblName; + } + } + return authzObj == null ? null : authzObj.toLowerCase(); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/2911c532/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index a35c8d7..dfaac15 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -35,6 +35,10 @@ import org.apache.sentry.core.model.db.AccessConstants; import org.apache.sentry.core.common.exception.SentryAlreadyExistsException; import org.apache.sentry.core.common.exception.SentryGrantDeniedException; import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; +import org.apache.sentry.hdfs.PermissionsUpdate; +import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; +import org.apache.sentry.hdfs.service.thrift.TRoleChanges; +import org.apache.sentry.provider.db.service.model.MSentryPermChange; import org.apache.sentry.provider.db.service.model.MSentryPrivilege; import org.apache.sentry.provider.db.service.model.MSentryRole; import org.apache.sentry.provider.db.service.thrift.TSentryActiveRoleSet; @@ -57,6 +61,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Files; +import static org.apache.sentry.hdfs.Updateable.Update; + public class TestSentryStore extends org.junit.Assert { private static File dataDir; @@ -2263,6 +2269,208 @@ public class TestSentryStore extends org.junit.Assert { assertTrue(names.containsAll(result)); } + @Test + public void testPrivilegesWithPermUpdate() throws Exception { + String roleName = "test-privilege"; + String grantor = "g1"; + String server = "server1"; + String db = "db1"; + String table = "tbl1"; + String authzObj = "db1.tbl1"; + createRole(roleName); + + TSentryPrivilege privilege = new TSentryPrivilege(); + privilege.setPrivilegeScope("Column"); + privilege.setServerName(server); + privilege.setDbName(db); + privilege.setTableName(table); + privilege.setAction(AccessConstants.SELECT); + privilege.setCreateTime(System.currentTimeMillis()); + + // Generate the permission add update authzObj "db1.tbl1" + PermissionsUpdate addUpdate = new PermissionsUpdate(0, false); + addUpdate.addPrivilegeUpdate(authzObj).putToAddPrivileges( + roleName, privilege.getAction().toUpperCase()); + + // Grant the privilege to role test-privilege and verify it has been persisted. + Map<TSentryPrivilege, Update> addPrivilegesUpdateMap = Maps.newHashMap(); + addPrivilegesUpdateMap.put(privilege, addUpdate); + sentryStore.alterSentryRoleGrantPrivileges(grantor, roleName, Sets.newHashSet(privilege), addPrivilegesUpdateMap); + MSentryRole role = sentryStore.getMSentryRoleByName(roleName); + Set<MSentryPrivilege> privileges = role.getPrivileges(); + assertEquals(privileges.toString(), 1, privileges.size()); + + // Query the persisted perm change and ensure it equals to the original one + long lastChangeID = sentryStore.getLastProcessedPermChangeID(); + MSentryPermChange addPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID); + assertEquals(addUpdate.JSONSerialize(), addPermChange.getPermChange()); + + // Generate the permission delete update authzObj "db1.tbl1" + PermissionsUpdate delUpdate = new PermissionsUpdate(0, false); + delUpdate.addPrivilegeUpdate(authzObj).putToDelPrivileges( + roleName, privilege.getAction().toUpperCase()); + + // Revoke the same privilege and verify it has been removed. + Map<TSentryPrivilege, Update> delPrivilegesUpdateMap = Maps.newHashMap(); + delPrivilegesUpdateMap.put(privilege, delUpdate); + sentryStore.alterSentryRoleRevokePrivileges(grantor, roleName, + Sets.newHashSet(privilege), delPrivilegesUpdateMap); + role = sentryStore.getMSentryRoleByName(roleName); + privileges = role.getPrivileges(); + assertEquals(0, privileges.size()); + + // Query the persisted perm change and ensure it equals to the original one + MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID + 1); + assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange()); + } + + @Test + public void testAddDeleteGroupsWithPermUpdate() throws Exception { + String roleName = "test-groups"; + String grantor = "g1"; + createRole(roleName); + + Set<TSentryGroup> groups = Sets.newHashSet(); + TSentryGroup group = new TSentryGroup(); + group.setGroupName("test-groups-g1"); + groups.add(group); + group = new TSentryGroup(); + group.setGroupName("test-groups-g2"); + groups.add(group); + + // Generate the permission add update for role "test-groups" + PermissionsUpdate addUpdate = new PermissionsUpdate(0, false); + TRoleChanges addrUpdate = addUpdate.addRoleUpdate(roleName); + for (TSentryGroup g : groups) { + addrUpdate.addToAddGroups(g.getGroupName()); + } + + // Assign the role "test-groups" to the groups and verify. + sentryStore.alterSentryRoleAddGroups(grantor, roleName, groups, addUpdate); + MSentryRole role = sentryStore.getMSentryRoleByName(roleName); + assertEquals(2, role.getGroups().size()); + + // Query the persisted perm change and ensure it equals to the original one + long lastChangeID = sentryStore.getLastProcessedPermChangeID(); + MSentryPermChange addPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID); + assertEquals(addUpdate.JSONSerialize(), addPermChange.getPermChange()); + + // Generate the permission add update for role "test-groups" + PermissionsUpdate delUpdate = new PermissionsUpdate(0, false); + TRoleChanges delrUpdate = delUpdate.addRoleUpdate(roleName); + for (TSentryGroup g : groups) { + delrUpdate.addToDelGroups(g.getGroupName()); + } + + // Revoke the role "test-groups" to the groups and verify. + sentryStore.alterSentryRoleDeleteGroups(roleName, groups, delUpdate); + role = sentryStore.getMSentryRoleByName(roleName); + assertEquals(Collections.emptySet(), role.getGroups()); + + // Query the persisted perm change and ensure it equals to the original one + MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID + 1); + assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange()); + } + + @Test + public void testCreateDropRoleWithPermUpdate() throws Exception { + String roleName = "test-drop-role"; + createRole(roleName); + + // Generate the permission del update for dropping role "test-drop-role" + PermissionsUpdate delUpdate = new PermissionsUpdate(0, false); + delUpdate.addPrivilegeUpdate(PermissionsUpdate.ALL_AUTHZ_OBJ).putToDelPrivileges( + roleName, PermissionsUpdate.ALL_AUTHZ_OBJ); + delUpdate.addRoleUpdate(roleName).addToDelGroups(PermissionsUpdate.ALL_GROUPS); + + // Drop the role and verify. + sentryStore.dropSentryRole(roleName, delUpdate); + checkRoleDoesNotExist(roleName); + + // Query the persisted perm change and ensure it equals to the original one + long lastChangeID = sentryStore.getLastProcessedPermChangeID(); + MSentryPermChange delPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID); + assertEquals(delUpdate.JSONSerialize(), delPermChange.getPermChange()); + } + + @Test + public void testDropObjWithPermUpdate() throws Exception { + String roleName1 = "list-privs-r1", roleName2 = "list-privs-r2"; + String grantor = "g1"; + sentryStore.createSentryRole(roleName1); + sentryStore.createSentryRole(roleName2); + + String authzObj = "db1.tbl1"; + TSentryPrivilege privilege_tbl1 = new TSentryPrivilege(); + privilege_tbl1.setPrivilegeScope("TABLE"); + privilege_tbl1.setServerName("server1"); + privilege_tbl1.setDbName("db1"); + privilege_tbl1.setTableName("tbl1"); + privilege_tbl1.setCreateTime(System.currentTimeMillis()); + privilege_tbl1.setAction("SELECT"); + + sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege_tbl1); + + // Generate the permission drop update for dropping privilege for "db1.tbl1" + PermissionsUpdate dropUpdate = new PermissionsUpdate(0, false); + dropUpdate.addPrivilegeUpdate(authzObj).putToDelPrivileges(PermissionsUpdate.ALL_ROLES, + PermissionsUpdate.ALL_ROLES); + + // Drop the privilege and verify. + sentryStore.dropPrivilege(toTSentryAuthorizable(privilege_tbl1), dropUpdate); + assertEquals(0, sentryStore.getAllTSentryPrivilegesByRoleName(roleName1).size()); + assertEquals(0, sentryStore.getAllTSentryPrivilegesByRoleName(roleName2).size()); + + // Query the persisted perm change and ensure it equals to the original one + long lastChangeID = sentryStore.getLastProcessedPermChangeID(); + MSentryPermChange dropPermChange = sentryStore.getMSentryPermChangeByID(lastChangeID); + assertEquals(dropUpdate.JSONSerialize(), dropPermChange.getPermChange()); + } + + @Test + public void testRenameObjWithPermUpdate() throws Exception { + String roleName1 = "role1", roleName2 = "role2", roleName3 = "role3"; + String grantor = "g1"; + String table1 = "tbl1", table2 = "tbl2"; + + sentryStore.createSentryRole(roleName1); + + TSentryPrivilege privilege_tbl1 = new TSentryPrivilege(); + privilege_tbl1.setPrivilegeScope("TABLE"); + privilege_tbl1.setServerName("server1"); + privilege_tbl1.setDbName("db1"); + privilege_tbl1.setTableName(table1); + privilege_tbl1.setCreateTime(System.currentTimeMillis()); + privilege_tbl1.setAction(AccessConstants.ALL); + + sentryStore.alterSentryRoleGrantPrivilege(grantor, roleName1, privilege_tbl1); + + // Generate the permission rename update for renaming privilege for "db1.tbl1" + String oldAuthz = "db1.tbl1"; + String newAuthz = "db1.tbl2"; + PermissionsUpdate renameUpdate = new PermissionsUpdate(0, false); + TPrivilegeChanges privUpdate = renameUpdate.addPrivilegeUpdate(PermissionsUpdate.RENAME_PRIVS); + privUpdate.putToAddPrivileges(newAuthz, newAuthz); + privUpdate.putToDelPrivileges(oldAuthz, oldAuthz); + + // Rename the privilege and verify. + TSentryAuthorizable oldTable = toTSentryAuthorizable(privilege_tbl1); + TSentryAuthorizable newTable = toTSentryAuthorizable(privilege_tbl1); + newTable.setTable(table2); + sentryStore.renamePrivilege(oldTable, newTable, renameUpdate); + + Set<TSentryPrivilege> privilegeSet = sentryStore.getAllTSentryPrivilegesByRoleName(roleName1); + assertEquals(1, privilegeSet.size()); + for (TSentryPrivilege privilege : privilegeSet) { + assertTrue(table2.equalsIgnoreCase(privilege.getTableName())); + } + + // Query the persisted perm change and ensure it equals to the original one + long lastChangeID = sentryStore.getLastProcessedPermChangeID(); + MSentryPermChange renamePermChange = sentryStore.getMSentryPermChangeByID(lastChangeID); + assertEquals(renameUpdate.JSONSerialize(), renamePermChange.getPermChange()); + } + protected static void addGroupsToUser(String user, String... groupNames) { policyFile.addGroupsToUser(user, groupNames); } http://git-wip-us.apache.org/repos/asf/sentry/blob/2911c532/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java index d601b1e..fd97936 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java @@ -17,13 +17,11 @@ package org.apache.sentry.service.thrift; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; -import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -56,8 +54,9 @@ public class TestHMSFollower { authorizable.setServer(hiveInstance); authorizable.setDb("db1"); - verify(sentryStore, times(1)).dropPrivilege(authorizable); + verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable)); } + @Test public void testDropDatabase() throws Exception { String dbName = "db1"; @@ -76,7 +75,7 @@ public class TestHMSFollower { authorizable.setServer(hiveInstance); authorizable.setDb("db1"); - verify(sentryStore, times(1)).dropPrivilege(authorizable); + verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable)) ; } @Test public void testCreateTable() throws Exception { @@ -100,7 +99,7 @@ public class TestHMSFollower { authorizable.setDb("db1"); authorizable.setTable(tableName); - verify(sentryStore, times(1)).dropPrivilege(authorizable); + verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable)); } @Test public void testDropTable() throws Exception { @@ -124,7 +123,7 @@ public class TestHMSFollower { authorizable.setDb("db1"); authorizable.setTable(tableName); - verify(sentryStore, times(1)).dropPrivilege(authorizable); + verify(sentryStore, times(1)).dropPrivilege(authorizable, HMSFollower.onDropSentryPrivilege(authorizable)); } @Test public void testRenameTable() throws Exception { @@ -160,6 +159,6 @@ public class TestHMSFollower { newAuthorizable.setDb(newDbName); newAuthorizable.setTable(newTableName); - verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable); + verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, HMSFollower.onRenameSentryPrivilege(authorizable, newAuthorizable)); } }
