Repository: sentry Updated Branches: refs/heads/master 596841949 -> da1863f34
http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java index f3f58f6..35cb2bb 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.sentry.binding.hive.authz.SentryHiveAuthorizerFactory; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars; import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider; @@ -129,8 +130,9 @@ public class HiveServerFactory { } if(!properties.containsKey(METASTORE_CONNECTION_URL)) { properties.put(METASTORE_CONNECTION_URL, - String.format("jdbc:derby:;databaseName=%s;create=true", + String.format("jdbc:derby:;databaseName=%s;create=true;createDatabaseIfNotExist=true", new File(baseDir, "metastore").getPath())); + properties.put("datanucleus.schema.autoCreateTables", "true"); } if(!properties.containsKey(ACCESS_TESTING_MODE)) { properties.put(ACCESS_TESTING_MODE, "true"); @@ -180,11 +182,20 @@ public class HiveServerFactory { properties.put(ConfVars.METASTORESERVERMINTHREADS.varname, "5"); } - // set the SentryMetaStoreFilterHook for HiveServer2 only, not for metastore - if (!HiveServer2Type.InternalMetastore.equals(type)) { - properties.put(ConfVars.METASTORE_FILTER_HOOK.varname, - org.apache.sentry.binding.metastore.SentryMetaStoreFilterHook.class.getName()); - } + properties.put(ConfVars.HIVE_AUTHORIZATION_ENABLED.varname, "false"); + properties.put(ConfVars.HIVE_AUTHORIZATION_MANAGER.varname, SentryHiveAuthorizerFactory.class.getName()); + + // CBO has a bug on Hive 2.0.0 with VIEWS because ReadIdentity objects are sent without + // parent information for partitioned columns + properties.put(ConfVars.HIVE_CBO_ENABLED.varname, "false"); + + // Hive 2.x set the following configuration to TRUE by default and it causes test issues on + // Sentry because we're trying to change columns with different column types + properties.put(ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES.varname, "false"); + + // This configuration will avoid starting the HS2 WebUI that was causing test failures when + // HS2 is configured for concurrency + properties.put(ConfVars.HIVE_IN_TEST.varname, "true"); if (!properties.containsKey(METASTORE_BYPASS)) { properties.put(METASTORE_BYPASS, "hive,impala," + System.getProperty("user.name", "")); http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java index e9b3a43..ea44e33 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java @@ -348,7 +348,7 @@ public class TestDBNotificationListenerInBuiltDeserializer extends AbstractMetas assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType()); assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName - assertEquals(partVals1, alterPartitionMessage.getValues()); + assertEquals(partVals1, alterPartitionMessage.getKeyValues()); if (alterPartitionMessage instanceof SentryJSONAlterPartitionMessage) { SentryJSONAlterPartitionMessage sjAlterPartitionMessage = (SentryJSONAlterPartitionMessage) alterPartitionMessage; assertEquals(partVals1, sjAlterPartitionMessage.getNewValues()); @@ -364,7 +364,7 @@ public class TestDBNotificationListenerInBuiltDeserializer extends AbstractMetas assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType()); assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName - assertEquals(partVals1, alterPartitionMessage.getValues()); + assertEquals(partVals1, alterPartitionMessage.getKeyValues()); if (alterPartitionMessage instanceof SentryJSONAlterPartitionMessage) { SentryJSONAlterPartitionMessage sjAlterPartitionMessage = (SentryJSONAlterPartitionMessage) alterPartitionMessage; assertEquals(partVals2, sjAlterPartitionMessage.getNewValues()); http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java index f43b316..f33b2f3 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java @@ -29,6 +29,7 @@ import org.hamcrest.text.IsEqualIgnoringCase; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import org.junit.*; @@ -374,7 +375,8 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW if(!useDbNotificationListener) { assertEquals(oldLocation.toLowerCase(), alterPartitionMessage.getOldLocation()); assertEquals(newLocation.toLowerCase(), alterPartitionMessage.getNewLocation()); - assertEquals(partVals1, alterPartitionMessage.getValues()); + assertTrue(partVals1.containsAll(alterPartitionMessage.getKeyValues().values()) && + alterPartitionMessage.getKeyValues().values().containsAll(partVals1)); assertEquals(partVals1, alterPartitionMessage.getNewValues()); } @@ -389,9 +391,12 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName if(!useDbNotificationListener) { - assertEquals(partVals1, alterPartitionMessage.getValues()); + assertTrue(partVals1.containsAll(alterPartitionMessage.getKeyValues().values()) && + alterPartitionMessage.getKeyValues().values().containsAll(partVals1)); assertEquals(partVals2, alterPartitionMessage.getNewValues()); } + + dropMetastoreDBIfExists(client, testDB); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties b/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties new file mode 100644 index 0000000..a6ddc29 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/resources/log4j2.properties @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Define some default values that can be overridden by system properties. +# +# For testing, it may also be convenient to specify + +# list of properties +property.sentry.root.logger = INFO + +# list of all appenders +appenders = console + +# console appender +appender.console.name = console +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d (%t) [%p - %l] %m%n + +# list of all loggers +loggers = Sentry, HadoopConf, Datanucleus + +logger.Sentry.name = org.apache.sentry +logger.Sentry.level = DEBUG + +logger.HadoopConf.name = org.apache.hadoop.conf.Configuration +logger.HadoopConf.level = ERROR + +logger.Datanucleus.name = org.datanucleus +logger.Datanucleus.level = ERROR + +# root logger +rootLogger.level = ${sys:sentry.root.logger} +rootLogger.appenderRefs = console +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.console.level = ${sys:sentry.root.logger} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-kafka/pom.xml b/sentry-tests/sentry-tests-kafka/pom.xml index 56a3ef1..c65323d 100644 --- a/sentry-tests/sentry-tests-kafka/pom.xml +++ b/sentry-tests/sentry-tests-kafka/pom.xml @@ -65,69 +65,25 @@ limitations under the License. <artifactId>curator-framework</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-core</artifactId> + <version>${datanucleus-core.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + <version>${datanucleus-api-jdo.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + <version>${datanucleus-rdbms.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + <version>${datanucleus-jdo.version}</version> + </dependency> </dependencies> - <profiles> - <profile> - <id>datanucleus3</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <properties> - <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version> - <datanucleus-core.version>3.2.12</datanucleus-core.version> - <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version> - </properties> - <dependencies> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-core</artifactId> - <version>${datanucleus-core.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-api-jdo</artifactId> - <version>${datanucleus-api-jdo.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-rdbms</artifactId> - <version>${datanucleus-rdbms.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>datanucleus4</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <datanucleus-api-jdo.version>4.2.1</datanucleus-api-jdo.version> - <datanucleus-core.version>4.1.6</datanucleus-core.version> - <datanucleus-rdbms.version>4.1.7</datanucleus-rdbms.version> - <datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version> - </properties> - <dependencies> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-core</artifactId> - <version>${datanucleus-core.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-api-jdo</artifactId> - <version>${datanucleus-api-jdo.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-rdbms</artifactId> - <version>${datanucleus-rdbms.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>javax.jdo</artifactId> - <version>${datanucleus-jdo.version}</version> - </dependency> - </dependencies> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-solr/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/pom.xml b/sentry-tests/sentry-tests-solr/pom.xml index c704768..efb708c 100644 --- a/sentry-tests/sentry-tests-solr/pom.xml +++ b/sentry-tests/sentry-tests-solr/pom.xml @@ -82,6 +82,26 @@ limitations under the License. </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-core</artifactId> + <version>${datanucleus-core.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + <version>${datanucleus-api-jdo.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + <version>${datanucleus-rdbms.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + <version>${datanucleus-jdo.version}</version> + </dependency> </dependencies> <build> <plugins> @@ -94,68 +114,4 @@ limitations under the License. </plugin> </plugins> </build> - <profiles> - <profile> - <id>datanucleus3</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <properties> - <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version> - <datanucleus-core.version>3.2.12</datanucleus-core.version> - <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version> - </properties> - <dependencies> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-core</artifactId> - <version>${datanucleus-core.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-api-jdo</artifactId> - <version>${datanucleus-api-jdo.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-rdbms</artifactId> - <version>${datanucleus-rdbms.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>datanucleus4</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <datanucleus-api-jdo.version>4.2.1</datanucleus-api-jdo.version> - <datanucleus-core.version>4.1.6</datanucleus-core.version> - <datanucleus-rdbms.version>4.1.7</datanucleus-rdbms.version> - <datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version> - </properties> - <dependencies> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-core</artifactId> - <version>${datanucleus-core.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-api-jdo</artifactId> - <version>${datanucleus-api-jdo.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-rdbms</artifactId> - <version>${datanucleus-rdbms.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>javax.jdo</artifactId> - <version>${datanucleus-jdo.version}</version> - </dependency> - </dependencies> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/sentry/blob/da1863f3/sentry-tests/sentry-tests-sqoop/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-sqoop/pom.xml b/sentry-tests/sentry-tests-sqoop/pom.xml index 9e7cee9..05b1653 100644 --- a/sentry-tests/sentry-tests-sqoop/pom.xml +++ b/sentry-tests/sentry-tests-sqoop/pom.xml @@ -95,71 +95,29 @@ limitations under the License. <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-core</artifactId> + <version>${datanucleus-core.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + <version>${datanucleus-api-jdo.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + <version>${datanucleus-rdbms.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + <version>${datanucleus-jdo.version}</version> + </dependency> </dependencies> <profiles> <profile> - <id>datanucleus3</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <properties> - <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version> - <datanucleus-core.version>3.2.12</datanucleus-core.version> - <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version> - </properties> - <dependencies> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-core</artifactId> - <version>${datanucleus-core.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-api-jdo</artifactId> - <version>${datanucleus-api-jdo.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-rdbms</artifactId> - <version>${datanucleus-rdbms.version}</version> - </dependency> - </dependencies> - </profile> - <profile> - <id>datanucleus4</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <datanucleus-api-jdo.version>4.2.1</datanucleus-api-jdo.version> - <datanucleus-core.version>4.1.6</datanucleus-core.version> - <datanucleus-rdbms.version>4.1.7</datanucleus-rdbms.version> - <datanucleus-jdo.version>3.2.0-m3</datanucleus-jdo.version> - </properties> - <dependencies> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-core</artifactId> - <version>${datanucleus-core.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-api-jdo</artifactId> - <version>${datanucleus-api-jdo.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>datanucleus-rdbms</artifactId> - <version>${datanucleus-rdbms.version}</version> - </dependency> - <dependency> - <groupId>org.datanucleus</groupId> - <artifactId>javax.jdo</artifactId> - <version>${datanucleus-jdo.version}</version> - </dependency> - </dependencies> - </profile> - <profile> <id>download-sqoop2</id> <activation> <activeByDefault>true</activeByDefault>
