Repository: sentry Updated Branches: refs/heads/master f45727ab1 -> aa87868ff
SENTRY-1447: When s3 is configured as HDFS defaultFS and Hive Warehouse Dir, need to fix some e2e test failures. For example, TestDbHdfsMaxGroups.java. Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/aa87868f Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/aa87868f Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/aa87868f Branch: refs/heads/master Commit: aa87868ff89b1991afed2401e92825cd34e2d6cb Parents: f45727a Author: Anne Yu <[email protected]> Authored: Tue Aug 23 12:16:41 2016 -0700 Committer: Anne Yu <[email protected]> Committed: Wed Aug 24 10:45:08 2016 -0700 ---------------------------------------------------------------------- sentry-tests/sentry-tests-hive/pom.xml | 1 + .../e2e/dbprovider/TestColumnEndToEnd.java | 62 --------- .../e2e/dbprovider/TestDbCrossOperations.java | 130 +++++++++++++++++++ .../sentry/tests/e2e/hdfs/TestDbHdfsBase.java | 6 + .../e2e/hive/TestExportImportPrivileges.java | 4 + .../tests/e2e/hive/TestOperationsPart2.java | 5 + 6 files changed, 146 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/aa87868f/sentry-tests/sentry-tests-hive/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/pom.xml b/sentry-tests/sentry-tests-hive/pom.xml index 2979b45..a2512ee 100644 --- a/sentry-tests/sentry-tests-hive/pom.xml +++ b/sentry-tests/sentry-tests-hive/pom.xml @@ -491,6 +491,7 @@ limitations under the License. <include>**/TestDbHdfsExtMaxGroups.java</include> <include>**/TestHiveWarehouseOnExtFs.java</include> <include>**/TestTableOnExtFS.java</include> + <include>**/TestDbCrossOperations.java</include> </includes> <argLine>-Dsentry.e2etest.hiveServer2Type=UnmanagedHiveServer2 -Dsentry.e2etest.DFSType=ClusterDFS -Dsentry.e2etest.external.sentry=true</argLine> </configuration> http://git-wip-us.apache.org/repos/asf/sentry/blob/aa87868f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java index 8c15257..22d4cf8 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestColumnEndToEnd.java @@ -414,66 +414,4 @@ public class TestColumnEndToEnd extends AbstractTestWithStaticConfiguration { statement.close(); connection.close(); } - - @Test - public void testCrossDbTableOperations() throws Exception { - //The privilege of user_role1 is used to test create table as select. - //The privilege of user_role2 is used to test create view as select. - Connection connection = context.createConnection(ADMIN1); - Statement statement = context.createStatement(connection); - statement.execute("CREATE database " + DB1); - statement.execute("CREATE database " + DB2); - statement.execute("use " + DB1); - statement.execute("CREATE ROLE user_role1"); - statement.execute("CREATE ROLE user_role2"); - statement.execute("CREATE TABLE tb1 (id int , name String)"); - statement.execute("GRANT CREATE ON DATABASE db_1 TO ROLE user_role1"); - statement.execute("GRANT CREATE ON DATABASE db_1 TO ROLE user_role2"); - statement.execute("GRANT SELECT (id) ON TABLE tb1 TO ROLE user_role1"); - statement.execute("GRANT SELECT ON TABLE tb1 TO ROLE user_role2"); - statement.execute("GRANT ROLE user_role1 TO GROUP " + USERGROUP1); - statement.execute("GRANT ROLE user_role2 TO GROUP " + USERGROUP2); - statement.execute("load data local inpath '" + dataFile.getPath() + "' into table db_1.tb1" ); - statement.execute("use " + DB2); - statement.execute("CREATE TABLE tb2 (id int, num String)"); - statement.execute("CREATE TABLE tb3 (id int, val String)"); - statement.execute("GRANT SELECT (num) ON TABLE tb2 TO ROLE user_role1"); - statement.execute("GRANT SELECT (val) ON TABLE tb3 TO ROLE user_role1"); - statement.execute("GRANT SELECT ON TABLE tb2 TO ROLE user_role2"); - statement.execute("GRANT SELECT ON TABLE tb3 TO ROLE user_role2"); - statement.execute("GRANT ROLE user_role1 TO GROUP " + USERGROUP1); - statement.execute("GRANT ROLE user_role2 TO GROUP " + USERGROUP2); - statement.execute("load data local inpath '" + dataFile.getPath() + "' into table db_2.tb2" ); - statement.execute("load data local inpath '" + dataFile.getPath() + "' into table db_2.tb3" ); - statement.close(); - connection.close(); - - connection =context.createConnection(USER1_1); - statement = context.createStatement(connection); - statement.execute("use " + DB1); - statement.execute("CREATE table db_1.t1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2"); - statement.close(); - connection.close(); - - connection =context.createConnection(USER2_1); - statement = context.createStatement(connection); - //The db_1.tb1 and db_2.tb3 is same with db_2.tb2. - ResultSet res = statement.executeQuery("select * from db_2.tb2 limit 2"); - List<String> expectedResult = new ArrayList<String>(); - List<String> returnedResult = new ArrayList<String>(); - expectedResult.add("238"); - expectedResult.add("86"); - while(res.next()){ - returnedResult.add(res.getString(1).trim()); - } - validateReturnedResult(expectedResult, returnedResult); - expectedResult.clear(); - returnedResult.clear(); - - statement.execute("use " + DB1); - statement.execute("CREATE VIEW db_1.v1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2"); - statement.close(); - connection.close(); - - } } http://git-wip-us.apache.org/repos/asf/sentry/blob/aa87868f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java new file mode 100644 index 0000000..62c0f58 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbCrossOperations.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.dbprovider; + +import java.io.File; +import java.io.FileOutputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.io.Resources; + +public class TestDbCrossOperations extends AbstractTestWithStaticConfiguration { + private static final Logger LOGGER = LoggerFactory. + getLogger(TestDbCrossOperations.class); + + private final String SINGLE_TYPE_DATA_FILE_NAME = "kv1.dat"; + private File dataFile; + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception{ + LOGGER.info("TestColumnView setupTestStaticConfiguration"); + useSentryService = true; + AbstractTestWithStaticConfiguration.setupTestStaticConfiguration(); + } + + @Override + @Before + public void setup() throws Exception { + super.setupAdmin(); + super.setup(); + super.setupPolicy(); + dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME); + FileOutputStream to = new FileOutputStream(dataFile); + Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); + to.close(); + } + + private void createTestData() throws Exception { + try (Connection connection = context.createConnection(ADMIN1)) { + try (Statement statement = context.createStatement(connection)) { + exec(statement, "CREATE database " + DB1); + exec(statement, "CREATE database " + DB2); + exec(statement, "use " + DB1); + exec(statement, "CREATE ROLE user_role1"); + exec(statement, "CREATE ROLE user_role2"); + exec(statement, "CREATE TABLE tb1 (id int , name String)"); + exec(statement, "GRANT ALL ON DATABASE db_1 TO ROLE user_role1"); + exec(statement, "GRANT ALL ON DATABASE db_1 TO ROLE user_role2"); + exec(statement, "GRANT SELECT (id) ON TABLE tb1 TO ROLE user_role1"); + exec(statement, "GRANT SELECT ON TABLE tb1 TO ROLE user_role2"); + exec(statement, "GRANT ROLE user_role1 TO GROUP " + USERGROUP1); + exec(statement, "GRANT ROLE user_role2 TO GROUP " + USERGROUP2); + exec(statement, "load data local inpath '" + dataFile.getPath() + "' into table db_1.tb1" ); + exec(statement, "use " + DB2); + exec(statement, "CREATE TABLE tb2 (id int, num String)"); + exec(statement, "CREATE TABLE tb3 (id int, val String)"); + exec(statement, "GRANT SELECT (num) ON TABLE tb2 TO ROLE user_role1"); + exec(statement, "GRANT SELECT (val) ON TABLE tb3 TO ROLE user_role1"); + exec(statement, "GRANT SELECT ON TABLE tb2 TO ROLE user_role2"); + exec(statement, "GRANT SELECT ON TABLE tb3 TO ROLE user_role2"); + exec(statement, "GRANT ROLE user_role1 TO GROUP " + USERGROUP1); + exec(statement, "GRANT ROLE user_role2 TO GROUP " + USERGROUP2); + exec(statement, "load data local inpath '" + dataFile.getPath() + "' into table db_2.tb2" ); + exec(statement, "load data local inpath '" + dataFile.getPath() + "' into table db_2.tb3" ); + } + } + } + + @Test + public void testCrossDbTableOperations() throws Exception { + //The privilege of user_role1 is used to test create table as select. + //The privilege of user_role2 is used to test create view as select. + createTestData(); + + //Test create table as select from cross db table + try (Connection connection =context.createConnection(USER1_1)) { + try (Statement statement = context.createStatement(connection)) { + statement.execute("use " + DB1); + statement.execute("CREATE table db_1.t1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2"); + } + } + + //Test create view as select from cross db tables + try (Connection connection =context.createConnection(USER2_1)) { + try (Statement statement = context.createStatement(connection)) { + //The db_1.tb1 and db_2.tb3 is same with db_2.tb2. + ResultSet res = statement.executeQuery("select * from db_2.tb2 limit 2"); + List<String> expectedResult = new ArrayList<String>(); + List<String> returnedResult = new ArrayList<String>(); + expectedResult.add("238"); + expectedResult.add("86"); + while(res.next()){ + returnedResult.add(res.getString(1).trim()); + } + validateReturnedResult(expectedResult, returnedResult); + expectedResult.clear(); + returnedResult.clear(); + exec(statement, "use " + DB1); + exec(statement, "CREATE VIEW db_1.v1 as select tb1.id, tb3.val, tb2.num from db_1.tb1,db_2.tb3,db_2.tb2"); + } + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/aa87868f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java index e545c37..17bc612 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestDbHdfsBase.java @@ -40,6 +40,7 @@ import static org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory.Hive import org.apache.sentry.tests.e2e.hive.fs.TestFSContants; import org.junit.After; +import org.junit.Assume; import org.junit.BeforeClass; import static org.hamcrest.Matchers.equalTo; @@ -96,9 +97,14 @@ public abstract class TestDbHdfsBase extends AbstractTestWithStaticConfiguration System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_TYPE, DFSType.MiniDFS.name()); protected final static String dfsAdmin = System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_ADMIN, "hdfs"); + protected final static String storageUriStr = System.getProperty(TestFSContants.SENTRY_E2E_TEST_STORAGE_URI); @BeforeClass public static void setupTestStaticConfiguration() throws Exception { + if (!Strings.isNullOrEmpty(storageUriStr)) { + LOGGER.warn("Skip HDFS tests if HDFS fileSystem is not configured on hdfs"); + Assume.assumeTrue(storageUriStr.toLowerCase().startsWith("hdfs")); + } useSentryService = true; enableHDFSAcls = true; AbstractTestWithStaticConfiguration.setupTestStaticConfiguration(); http://git-wip-us.apache.org/repos/asf/sentry/blob/aa87868f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java index b38eaaf..a02a156 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java @@ -131,6 +131,8 @@ public class TestExportImportPrivileges extends AbstractTestWithStaticConfigurat connection = context.createConnection(USER1_1); statement = context.createStatement(connection); statement.execute("use " + DB1); + // Import/Export works with s3 storage system only when this is turned on. + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); statement.execute("EXPORT TABLE " + TBL1 + " TO '" + exportDir + "'"); statement.close(); connection.close(); @@ -139,6 +141,7 @@ public class TestExportImportPrivileges extends AbstractTestWithStaticConfigurat connection = context.createConnection(USER2_1); statement = context.createStatement(connection); statement.execute("use " + DB1); + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); context.assertAuthzException(statement, "IMPORT TABLE " + TBL2 + " FROM '" + exportDir + "'"); statement.close(); connection.close(); @@ -147,6 +150,7 @@ public class TestExportImportPrivileges extends AbstractTestWithStaticConfigurat connection = context.createConnection(USER1_1); statement = context.createStatement(connection); statement.execute("use " + DB1); + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); statement.execute("IMPORT TABLE " + TBL2 + " FROM '" + exportDir + "'"); statement.close(); connection.close(); http://git-wip-us.apache.org/repos/asf/sentry/blob/aa87868f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java index 7186f1a..e11a817 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperationsPart2.java @@ -442,6 +442,8 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration { connection = context.createConnection(USER3_1); statement = context.createStatement(connection); statement.execute("Use " + DB1); + // Import/Export works with s3 storage system only when this is turned on. + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); context.assertSentrySemanticException(statement, "export table tb1 to '" + location + "'", semanticException); statement.close(); @@ -451,6 +453,7 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration { connection = context.createConnection(USER1_1); statement = context.createStatement(connection); statement.execute("Use " + DB1); + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); statement.execute("export table tb1 to '" + location + "'" ); statement.close(); connection.close(); @@ -459,6 +462,7 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration { connection = context.createConnection(USER3_1); statement = context.createStatement(connection); statement.execute("Use " + DB1); + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); context.assertSentrySemanticException(statement, "import table tb2 from '" + location + "'", semanticException); statement.close(); @@ -468,6 +472,7 @@ public class TestOperationsPart2 extends AbstractTestWithStaticConfiguration { connection = context.createConnection(USER2_1); statement = context.createStatement(connection); statement.execute("Use " + DB1); + exec(statement, "set hive.exim.uri.scheme.whitelist=hdfs,pfile,s3a;"); statement.execute("import table tb2 from '" + location + "'"); statement.close(); connection.close();
