Repository: incubator-sentry Updated Branches: refs/heads/master 9fe720232 -> 4b33ad929
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java index c9accc1..c8d56be 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationInfo.java @@ -17,9 +17,7 @@ */ package org.apache.sentry.hdfs; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -126,6 +124,7 @@ public class SentryAuthorizationInfo implements Runnable { } private boolean update() { + //Looks like getting same updates multiple times SentryAuthzUpdate updates = updater.getUpdates(); // Updates can be null if Sentry Service is un-reachable if (updates != null) { @@ -274,13 +273,24 @@ public class SentryAuthorizationInfo implements Runnable { public List<AclEntry> getAclEntries(String[] pathElements) { lock.readLock().lock(); try { - String authzObj = authzPaths.findAuthzObject(pathElements); + Set<String> authzObjs = authzPaths.findAuthzObject(pathElements); // Apparently setFAcl throws error if 'group::---' is not present AclEntry noGroup = AclEntry.parseAclEntry("group::---", true); - ArrayList<AclEntry> retList = Lists.newArrayList(noGroup); - retList.addAll((authzObj != null) ? authzPermissions.getAcls(authzObj) - : Collections.EMPTY_LIST); - return retList; + + Set<AclEntry> retSet = new HashSet<AclEntry>(); + retSet.add(noGroup); + + if (authzObjs == null) { + retSet.addAll(Collections.EMPTY_LIST); + return new ArrayList<AclEntry>(retSet); + } + + // No duplicate acls should be added. + for (String authzObj: authzObjs) { + retSet.addAll(authzPermissions.getAcls(authzObj)); + } + + return new ArrayList<AclEntry>(retSet); } finally { lock.readLock().unlock(); } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java index 2c50ea9..daa87cf 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryPermissions.java @@ -17,13 +17,7 @@ */ package org.apache.sentry.hdfs; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; @@ -79,9 +73,12 @@ public class SentryPermissions implements AuthzPermissions { } } - private final Map<String, PrivilegeInfo> privileges = new HashMap<String, PrivilegeInfo>(); + // Comparison of authorizable object should be case insensitive. + private final Map<String, PrivilegeInfo> privileges = new TreeMap<String, PrivilegeInfo>(String.CASE_INSENSITIVE_ORDER); + private Map<String, Set<String>> authzObjChildren = new TreeMap<String, Set<String>>(String.CASE_INSENSITIVE_ORDER); + + // Should the comparison of role be case insensitive? private final Map<String, RoleInfo> roles = new HashMap<String, RoleInfo>(); - private Map<String, Set<String>> authzObjChildren = new HashMap<String, Set<String>>(); String getParentAuthzObject(String authzObject) { int dot = authzObject.indexOf('.'); http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java index 437ba94..2c9e19d 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestMetastoreCacheInitializer.java @@ -29,6 +29,8 @@ import org.junit.Test; import org.mockito.Mockito; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; public class TestMetastoreCacheInitializer { @@ -113,19 +115,19 @@ public class TestMetastoreCacheInitializer { MetastoreCacheInitializer(hmsHandler, conf); UpdateableAuthzPaths update = cacheInitializer.createInitialUpdate(); - Assert.assertEquals("db1", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db1")), update.findAuthzObjectExactMatches(new String[]{"db1"})); - Assert.assertEquals("db2", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db2")), update.findAuthzObjectExactMatches(new String[]{"db2"})); - Assert.assertEquals("db2.tab21", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db2.tab21")), update.findAuthzObjectExactMatches(new String[]{"db2", "tab21"})); - Assert.assertEquals("db3", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db3")), update.findAuthzObjectExactMatches(new String[]{"db3"})); - Assert.assertEquals("db3.tab31", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new String[]{"db3", "tab31"})); - Assert.assertEquals("db3.tab31", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new String[]{"db3", "tab31", "part311"})); - Assert.assertEquals("db3.tab31", update.findAuthzObjectExactMatch(new + Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new String[]{"db3", "tab31", "part312"})); cacheInitializer.close(); http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/4b33ad92/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java index 208c93b..5a93ba0 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java @@ -76,8 +76,8 @@ import org.apache.hadoop.mapred.TextOutputFormat; import org.apache.hadoop.security.UserGroupInformation; import org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; -import org.apache.sentry.hdfs.SentryAuthorizationConstants; import org.apache.sentry.hdfs.SentryAuthorizationProvider; +import org.apache.sentry.provider.db.SentryAlreadyExistsException; import org.apache.sentry.provider.db.SimpleDBProviderBackend; import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider; import org.apache.sentry.provider.file.PolicyFile; @@ -1124,6 +1124,203 @@ public class TestHDFSIntegration { conn.close(); } + /* SENTRY-953 */ + @Test + public void testAuthzObjOnPartitionMultipleTables() throws Throwable { + String dbName = "db1"; + + tmpHDFSDir = new Path("/tmp/external"); + miniDFS.getFileSystem().mkdirs(tmpHDFSDir); + Path partitionDir = new Path("/tmp/external/p1"); + miniDFS.getFileSystem().mkdirs(partitionDir); + + dbNames = new String[]{dbName}; + roles = new String[]{"admin_role", "tab1_role", "tab2_role", "tab3_role"}; + admin = StaticUserGroup.ADMIN1; + + Connection conn; + Statement stmt; + + conn = hiveServer2.createConnection("hive", "hive"); + stmt = conn.createStatement(); + + stmt.execute("create role admin_role"); + stmt.execute("grant all on server server1 to role admin_role"); + stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); + + // Create external table tab1 on location '/tmp/external/p1'. + // Create tab1_role, and grant it with insert permission on table tab1 to user_group1. + conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); + stmt = conn.createStatement(); + stmt.execute("create database " + dbName); + stmt.execute("use " + dbName); + stmt.execute("create external table tab1 (s string) partitioned by (month int) location '/tmp/external/p1'"); + stmt.execute("create role tab1_role"); + stmt.execute("grant insert on table tab1 to role tab1_role"); + stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + + // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'. + verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); + + // Create external table tab2 and partition on location '/tmp/external'. + // Create tab2_role, and grant it with select permission on table tab2 to user_group2. + stmt.execute("create external table tab2 (s string) partitioned by (month int)"); + stmt.execute("alter table tab2 add partition (month = 1) location '/tmp/external'"); + stmt.execute("create role tab2_role"); + stmt.execute("grant select on table tab2 to role tab2_role"); + stmt.execute("grant role tab2_role to group " + StaticUserGroup.USERGROUP2); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + + // Verify that user_group2 have select(read_execute) permission on both paths. + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab2", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); + verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); + + // Create table tab3 and partition on the same location '/tmp/external' as tab2. + // Create tab3_role, and grant it with insert permission on table tab3 to user_group3. + stmt.execute("create table tab3 (s string) partitioned by (month int)"); + stmt.execute("alter table tab3 add partition (month = 1) location '/tmp/external'"); + stmt.execute("create role tab3_role"); + stmt.execute("grant insert on table tab3 to role tab3_role"); + stmt.execute("grant role tab3_role to group " + StaticUserGroup.USERGROUP3); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + + // When two partitions of different tables pointing to the same location with different grants, + // ACLs should have union (no duplicates) of both rules. + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); + verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + + // When alter the table name (tab2 to be tabx), ACLs should remain the same. + stmt.execute("alter table tab2 rename to tabx"); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + verifyOnPath("/tmp/external", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true); + verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + + // When drop a partition that shares the same location with other partition belonging to + // other table, should still have the other table permissions. + stmt.execute("ALTER TABLE tabx DROP PARTITION (month = 1)"); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + + // When drop a table that has a partition shares the same location with other partition + // belonging to other table, should still have the other table permissions. + stmt.execute("DROP TABLE IF EXISTS tabx"); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/tab3", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + verifyOnPath("/tmp/external", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP3, true); + + stmt.close(); + conn.close(); + + miniDFS.getFileSystem().delete(partitionDir, true); + } + + /* SENTRY-953 */ + @Test + public void testAuthzObjOnPartitionSameTable() throws Throwable { + String dbName = "db1"; + + tmpHDFSDir = new Path("/tmp/external/p1"); + miniDFS.getFileSystem().mkdirs(tmpHDFSDir); + + dbNames = new String[]{dbName}; + roles = new String[]{"admin_role", "tab1_role"}; + admin = StaticUserGroup.ADMIN1; + + Connection conn; + Statement stmt; + + conn = hiveServer2.createConnection("hive", "hive"); + stmt = conn.createStatement(); + + stmt.execute("create role admin_role"); + stmt.execute("grant all on server server1 to role admin_role"); + stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); + + // Create table tab1 and partition on the same location '/tmp/external/p1'. + // Create tab1_role, and grant it with insert permission on table tab1 to user_group1. + conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); + stmt = conn.createStatement(); + stmt.execute("create database " + dbName); + stmt.execute("use " + dbName); + stmt.execute("create table tab1 (s string) partitioned by (month int)"); + stmt.execute("alter table tab1 add partition (month = 1) location '/tmp/external/p1'"); + stmt.execute("create role tab1_role"); + stmt.execute("grant insert on table tab1 to role tab1_role"); + stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + + // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'. + verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); + + // When two partitions of the same table pointing to the same location, + // ACLS should not be repeated. Exception will be thrown if there are duplicates. + stmt.execute("alter table tab1 add partition (month = 2) location '/tmp/external/p1'"); + verifyOnPath("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); + + stmt.close(); + conn.close(); + } + + /* SENTRY-953 */ + @Test + public void testAuthzObjOnMultipleTables() throws Throwable { + String dbName = "db1"; + + tmpHDFSDir = new Path("/tmp/external/p1"); + miniDFS.getFileSystem().mkdirs(tmpHDFSDir); + + dbNames = new String[]{dbName}; + roles = new String[]{"admin_role", "tab1_role", "tab2_role"}; + admin = StaticUserGroup.ADMIN1; + + Connection conn; + Statement stmt; + + conn = hiveServer2.createConnection("hive", "hive"); + stmt = conn.createStatement(); + + stmt.execute("create role admin_role"); + stmt.execute("grant all on server server1 to role admin_role"); + stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP); + + // Create external table tab1 on location '/tmp/external/p1'. + // Create tab1_role, and grant it with insert permission on table tab1 to user_group1. + conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1); + stmt = conn.createStatement(); + stmt.execute("create database " + dbName); + stmt.execute("use " + dbName); + stmt.execute("create external table tab1 (s string) partitioned by (month int) location '/tmp/external/p1'"); + stmt.execute("create role tab1_role"); + stmt.execute("grant insert on table tab1 to role tab1_role"); + stmt.execute("grant role tab1_role to group " + StaticUserGroup.USERGROUP1); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + + // Verify that user_group1 has insert(write_execute) permission on '/tmp/external/p1'. + verifyOnAllSubDirs("/tmp/external/p1", FsAction.WRITE_EXECUTE, StaticUserGroup.USERGROUP1, true); + + // Create table tab2 on the same location '/tmp/external/p1' as table tab1. + // Create tab2_role, and grant it with select permission on table tab2 to user_group1. + stmt.execute("create table tab2 (s string) partitioned by (month int) location '/tmp/external/p1'"); + stmt.execute("create role tab2_role"); + stmt.execute("grant select on table tab2 to role tab2_role"); + stmt.execute("grant role tab2_role to group " + StaticUserGroup.USERGROUP1); + + // When two tables pointing to the same location, ACLS should have union (no duplicates) + // of both rules. + verifyOnPath("/tmp/external/p1", FsAction.ALL, StaticUserGroup.USERGROUP1, true); + + // When drop table tab1, ACLs of tab2 still remain. + stmt.execute("DROP TABLE IF EXISTS tab1"); + Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode + verifyOnPath("/tmp/external/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP1, true); + + stmt.close(); + conn.close(); + } + private void verifyAccessToPath(String user, String group, String path, boolean hasPermission) throws Exception{ Path p = new Path(path); UserGroupInformation hadoopUser = @@ -1305,7 +1502,13 @@ public class TestHDFSIntegration { Map<String, FsAction> acls = new HashMap<String, FsAction>(); for (AclEntry ent : aclStatus.getEntries()) { if (ent.getType().equals(AclEntryType.GROUP)) { - acls.put(ent.getName(), ent.getPermission()); + + // In case of duplicate acl exist, exception should be thrown. + if (acls.containsKey(ent.getName())) { + throw new SentryAlreadyExistsException("The acl " + ent.getName() + " already exists.\n"); + } else { + acls.put(ent.getName(), ent.getPermission()); + } } } return acls;
