SENTRY-2048: Bump Hive version to 2.3.2 (Sergio Pena, reviewed by kalyan kumar kalvagadda, Na Li)
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/5f64fe9f Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/5f64fe9f Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/5f64fe9f Branch: refs/heads/akolb-cli Commit: 5f64fe9f34fb857ccca5a5b09b9ff46300c98674 Parents: a355979 Author: Sergio Pena <[email protected]> Authored: Sun Nov 19 10:47:09 2017 -0600 Committer: Sergio Pena <[email protected]> Committed: Sun Nov 19 10:47:09 2017 -0600 ---------------------------------------------------------------------- pom.xml | 2 +- .../json/SentryJSONMessageDeserializer.java | 79 +++++++++++++++++++- .../json/SentryJSONMessageFactory.java | 48 ++++++++++++ .../hive/ql/exec/SentryFilterDDLTask.java | 9 ++- .../hive/ql/exec/SentryGrantRevokeTask.java | 7 +- .../binding/hive/HiveAuthzBindingHook.java | 2 +- .../hive/authz/DefaultSentryValidator.java | 17 +++++ .../hive/authz/HiveAuthzBindingHookBase.java | 10 --- .../hive/authz/SentryHiveAuthorizerImpl.java | 13 ++++ .../metastore/SentryHiveMetaStoreClient.java | 2 +- .../TestSentryHiveAuthorizationTaskFactory.java | 16 ++-- .../e2e/hdfs/TestHDFSIntegrationAdvanced.java | 9 ++- .../tests/e2e/hdfs/TestHDFSIntegrationBase.java | 4 + .../e2e/hive/TestPrivilegesAtColumnScope.java | 31 ++++---- .../e2e/hive/hiveserver/HiveServerFactory.java | 10 +++ ...actMetastoreTestWithStaticConfiguration.java | 3 +- .../e2e/metastore/TestMetastoreEndToEnd.java | 6 +- sentry-tests/sentry-tests-solr/pom.xml | 8 ++ 18 files changed, 230 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6487a26..d863627 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ limitations under the License. <guava.version>14.0.1</guava.version> <hadoop.version>2.7.2</hadoop.version> <hamcrest.version>1.3</hamcrest.version> - <hive.version>2.0.0</hive.version> + <hive.version>2.3.2</hive.version> <jackson.version>1.8.8</jackson.version> <jdo-api.version>3.0.1</jdo-api.version> <jetty.version>9.3.21.v20170918</jetty.version> http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java index d11b261..929ac8c 100644 --- a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java @@ -19,6 +19,11 @@ package org.apache.sentry.binding.metastore.messaging.json; import org.apache.hive.hcatalog.messaging.*; +import org.apache.hive.hcatalog.messaging.json.JSONAlterIndexMessage; +import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage; +import org.apache.hive.hcatalog.messaging.json.JSONCreateIndexMessage; +import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage; +import org.apache.hive.hcatalog.messaging.json.JSONDropIndexMessage; import org.apache.hive.hcatalog.messaging.json.JSONInsertMessage; import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; @@ -130,13 +135,81 @@ public class SentryJSONMessageDeserializer extends MessageDeserializer { } /** + * Method to de-serialize CreateFunctionMessage instance. + */ + @Override + public CreateFunctionMessage getCreateFunctionMessage(String messageBody) { + // Sentry does not need this message, but it needs to be implemented so that Hive can + // complete the notification log for such event. + try { + return mapper.readValue(messageBody, JSONCreateFunctionMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct JSONCreateFunctionMessage: ", e); + } + } + + /** + * Method to de-serialize DropFunctionMessage instance. + */ + @Override + public DropFunctionMessage getDropFunctionMessage(String messageBody) { + // Sentry does not need this message, but it needs to be implemented so that Hive can + // complete the notification log for such event. + try { + return mapper.readValue(messageBody, JSONDropFunctionMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct JSONDropDatabaseMessage: ", e); + } + } + + /** + * Method to de-serialize CreateIndexMessage instance. + */ + @Override + public CreateIndexMessage getCreateIndexMessage(String messageBody) { + // Sentry does not need this message, but it needs to be implemented so that Hive can + // complete the notification log for such event. + try { + return mapper.readValue(messageBody, JSONCreateIndexMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct JSONCreateIndexMessage: ", e); + } + } + + /** + * Method to de-serialize DropIndexMessage instance. + */ + @Override + public DropIndexMessage getDropIndexMessage(String messageBody) { + // Sentry does not need this message, but it needs to be implemented so that Hive can + // complete the notification log for such event. + try { + return mapper.readValue(messageBody, JSONDropIndexMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct JSONDropIndexMessage: ", e); + } + } + + /** + * Method to de-serialize AlterIndexMessage instance. + */ + @Override + public AlterIndexMessage getAlterIndexMessage(String messageBody) { + // Sentry does not need this message, but it needs to be implemented so that Hive can + // complete the notification log for such event. + try { + return mapper.readValue(messageBody, JSONAlterIndexMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct JSONAlterIndexMessage: ", e); + } + } + + /** * Method to de-serialize JSONInsertMessage instance. */ @Override public InsertMessage getInsertMessage(String messageBody) { - // Sentry would be not be interested in InsertMessage as these are generated when is data is - // added inserted. This method is implemented for completeness. This is reason why, new sentry - // JSON class is not defined for InsertMessage. + // Sentry does not need this message, but it needs to be implemented so that Hive can + // complete the notification log for such event. try { return mapper.readValue(messageBody, JSONInsertMessage.class); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java index b531976..0af02d1 100644 --- a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java @@ -22,9 +22,16 @@ import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.hcatalog.messaging.*; +import org.apache.hive.hcatalog.messaging.json.JSONAlterIndexMessage; +import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage; +import org.apache.hive.hcatalog.messaging.json.JSONCreateIndexMessage; +import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage; +import org.apache.hive.hcatalog.messaging.json.JSONDropIndexMessage; import org.apache.hive.hcatalog.messaging.json.JSONInsertMessage; import java.util.*; @@ -108,6 +115,47 @@ public class SentryJSONMessageFactory extends MessageFactory { } @Override + public CreateFunctionMessage buildCreateFunctionMessage(Function function) { + // Sentry would be not be interested in CreateFunctionMessage as these are generated when is data is + // added inserted. This method is implemented for completeness. This is reason why, new sentry + // JSON class is not defined for CreateFunctionMessage + return new JSONCreateFunctionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, function, now()); + } + + @Override + public DropFunctionMessage buildDropFunctionMessage(Function function) { + // Sentry would be not be interested in DropFunctionMessage as these are generated when is data is + // added inserted. This method is implemented for completeness. This is reason why, new sentry + // JSON class is not defined for DropFunctionMessage + return new JSONDropFunctionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, function, now()); + + } + + @Override + public CreateIndexMessage buildCreateIndexMessage(Index index) { + // Sentry would be not be interested in CreateIndexMessage as these are generated when is data is + // added inserted. This method is implemented for completeness. This is reason why, new sentry + // JSON class is not defined for CreateIndexMessage + return new JSONCreateIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, index, now()); + } + + @Override + public DropIndexMessage buildDropIndexMessage(Index index) { + // Sentry would be not be interested in DropIndexMessage as these are generated when is data is + // added inserted. This method is implemented for completeness. This is reason why, new sentry + // JSON class is not defined for DropIndexMessage + return new JSONDropIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, index, now()); + } + + @Override + public AlterIndexMessage buildAlterIndexMessage(Index before, Index after) { + // Sentry would be not be interested in AlterIndexMessage as these are generated when is data is + // added inserted. This method is implemented for completeness. This is reason why, new sentry + // JSON class is not defined for AlterIndexMessage + return new JSONAlterIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before, after, now()); + } + + @Override public InsertMessage buildInsertMessage(String db, String table, Map<String,String> partKeyVals, List<String> files) { // Sentry would be not be interested in InsertMessage as these are generated when is data is http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java index 672acb6..efe113e 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryFilterDDLTask.java @@ -143,9 +143,6 @@ public class SentryFilterDDLTask extends DDLTask { parentTasks = ddlTask.getParentTasks(); backupTask = ddlTask.getBackupTask(); backupChildrenTasks = ddlTask.getBackupChildrenTasks(); - started = ddlTask.started(); - isdone = ddlTask.done(); - queued = ddlTask.getQueued(); id = ddlTask.getId(); taskCounters = ddlTask.getCounters(); feedSubscribers = ddlTask.getFeedSubscribers(); @@ -157,5 +154,11 @@ public class SentryFilterDDLTask extends DDLTask { setException(ddlTask.getException()); console = ddlTask.console; setFetchSource(ddlTask.isFetchSource()); + taskHandle = ddlTask.getTaskHandle(); + conf = ddlTask.conf; + queryState = ddlTask.queryState; + driverContext = ddlTask.getDriverContext(); + clonedConf = ddlTask.clonedConf; + queryDisplay = ddlTask.queryDisplay; } } http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java index 61052cf..21a6abf 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/hadoop/hive/ql/exec/SentryGrantRevokeTask.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; @@ -109,11 +110,11 @@ public class SentryGrantRevokeTask extends Task<DDLWork> implements Serializable private HiveOperation stmtOperation; @Override - public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx, + public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx, CompilationOpContext opContext) { // CompilationOpContext is an unused parameter on the initialize() method. - super.initialize(conf, queryPlan, driverContext, null); - this.conf = conf; + super.initialize(queryState, queryPlan, driverContext, null); + this.conf = queryState.getConf(); } @Override http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java index 802bf9c..e4620ea 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java @@ -291,7 +291,7 @@ public class HiveAuthzBindingHook extends HiveAuthzBindingHookBase { @Override public void postAnalyze(HiveSemanticAnalyzerHookContext context, List<Task<? extends Serializable>> rootTasks) throws SemanticException { - HiveOperation stmtOperation = getCurrentHiveStmtOp(); + HiveOperation stmtOperation = context.getHiveOperation(); HiveAuthzPrivileges stmtAuthObject; stmtAuthObject = HiveAuthzPrivilegesMap.getHiveAuthzPrivileges(stmtOperation); http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java index d1f071e..319a1be 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/DefaultSentryValidator.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; @@ -375,6 +376,22 @@ public class DefaultSentryValidator extends SentryHiveAuthorizationValidator { return listObjs; } + @Override + public List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext hiveAuthzContext, + List<HivePrivilegeObject> list) throws SemanticException { + // Sentry does not support this feature yet. Returning null is enough to let Hive + // that no row filtering nor column masking will be applied. + return null; + } + + @Override + public boolean needTransform() { + // Hive uses this value to know whether a Hive query must be transformed if row filtering + // or column masking is applied. Sentry does not support such feature yet, so returning + // false is enough to let Hive know that the query is not required to be transformed. + return false; + } + private List<HivePrivilegeObject> filterShowTables(List<HivePrivilegeObject> listObjs, String userName, HiveAuthzBinding hiveAuthzBinding) { List<HivePrivilegeObject> filteredResult = new ArrayList<HivePrivilegeObject>(); http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java index 2e299a9..9c60c22 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzBindingHookBase.java @@ -886,18 +886,8 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH } } - protected HiveOperation getCurrentHiveStmtOp() { - SessionState sessState = SessionState.get(); - if (sessState == null) { - // TODO: Warn - return null; - } - return sessState.getHiveOperation(); - } - protected Subject getCurrentSubject(HiveSemanticAnalyzerHookContext context) { // Extract the username from the hook context return new Subject(context.getUserName()); } - } http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java index 1596bce..86ff0cc 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryHiveAuthorizerImpl.java @@ -20,11 +20,13 @@ import java.util.List; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.SentryHivePrivilegeObjectDesc; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc; import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationTranslator; import org.apache.hadoop.hive.ql.security.authorization.plugin.AbstractHiveAuthorizer; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizationTranslator; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer.VERSION; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; @@ -165,6 +167,17 @@ public class SentryHiveAuthorizerImpl extends AbstractHiveAuthorizer { return hiveTranslator; } + @Override + public List<HivePrivilegeObject> applyRowFilterAndColumnMasking(HiveAuthzContext hiveAuthzContext, + List<HivePrivilegeObject> list) throws SemanticException { + return null; + } + + @Override + public boolean needTransform() { + return false; + } + protected static HivePrivilegeObjectType getPrivObjectType( SentryHivePrivilegeObjectDesc privSubjectDesc) { if (privSubjectDesc.getObject() == null) { http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java index cfc5c04..e30a860 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/SentryHiveMetaStoreClient.java @@ -49,7 +49,7 @@ public class SentryHiveMetaStoreClient extends HiveMetaStoreClient implements public SentryHiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader) throws MetaException { - super(conf, hookLoader); + super(conf, hookLoader, true); } @Override http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java index de073ed..c75f57d 100644 --- a/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java +++ b/sentry-binding/sentry-binding-hive/src/test/java/org/apache/sentry/binding/hive/TestSentryHiveAuthorizationTaskFactory.java @@ -22,6 +22,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.List; +import org.apache.hadoop.hive.ql.QueryState; import org.junit.Assert; import org.apache.commons.io.FileUtils; @@ -37,7 +38,6 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; -import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -70,7 +70,6 @@ public class TestSentryHiveAuthorizationTaskFactory { private static final String SERVER = "server1"; - private ParseDriver parseDriver; private DDLSemanticAnalyzer analyzer; private HiveConf conf; private Context context; @@ -91,12 +90,19 @@ public class TestSentryHiveAuthorizationTaskFactory { conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY, SentryHiveAuthorizationTaskFactoryImpl.class.getName()); + // This configuration avoids starting the HS2 WebUI which was causes test failures when + // HS2 is configured for concurrency + conf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true); + + // This configuration avoids that the HMS fails if the Metastore schema has not version + // information. For some reason, HMS does not set a version initially on our tests. + conf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, false); + db = Mockito.mock(Hive.class); table = new Table(DB, TABLE); partition = new Partition(table); context = new Context(conf); - parseDriver = new ParseDriver(); - analyzer = new DDLSemanticAnalyzer(conf, db); + analyzer = new DDLSemanticAnalyzer(new QueryState(conf), db); SessionState.start(conf); Mockito.when(db.getTable(TABLE, false)).thenReturn(table); Mockito.when(db.getPartition(table, new HashMap<String, String>(), false)) @@ -487,7 +493,7 @@ public class TestSentryHiveAuthorizationTaskFactory { } private ASTNode parse(String command) throws Exception { - return ParseUtils.findRootNonNullToken(parseDriver.parse(command)); + return ParseUtils.parse(command); } private DDLWork analyze(ASTNode ast) throws Exception { http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java index 95bbaeb..33ace57 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationAdvanced.java @@ -791,7 +791,14 @@ public class TestHDFSIntegrationAdvanced extends TestHDFSIntegrationBase { // Alter table tab1 to be tbCopy which is at scheme-less location. // And the corresponding path will be updated to sentry server. hmsClient.alter_table(dbName, "tab1", tbCopy); - Assert.assertEquals(hmsClient.getTable(dbName, tblName).getSd().getLocation(), "/tmp/external"); + + // Remove the checking for the location of the table. The HMS will never return scheme-less + // URI locations anymore. However, if any NPE being triggered in future because of any changes, + // the test case will cover it and capture it. + // i.e. hdfs://<localhost>/tmp/external (location with scheme) + // /tmp/external (location without scheme) + // Assert.assertEquals("/tmp/external", hmsClient.getTable(dbName, tblName).getSd().getLocation()); + verifyOnPath("/tmp/external", FsAction.ALL, StaticUserGroup.HIVE, true); stmt.close(); http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java index 7495fc7..25a678b 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegrationBase.java @@ -575,6 +575,10 @@ public abstract class TestHDFSIntegrationBase { String hadoopTempDir = System.getProperty("java.io.tmpdir") + File.separator + "hadoop-tmp"; hiveConf.set("hadoop.tmp.dir", hadoopTempDir); + // This configuration will avoid that the HMS fails if the metastore schema has not version + // information. For some reason, HMS does not set a version initially on our tests. + hiveConf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false"); + // Sets hive.metastore.authorization.storage.checks to true, so that // disallow the operations such as drop-partition if the user in question // doesn't have permissions to delete the corresponding directory http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java index c2fee2a..9d820a8 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtColumnScope.java @@ -205,12 +205,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura context.verifyAuthzException(e); } - // negative test: test user can't query column of view + // positive test: test user can query column of view + // Hive 2 adds a new column view authorization as part of the Column Pruning feature + // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled try { statement.execute("SELECT COUNT(A) FROM VIEW_1"); - Assert.fail("Expected SQL exception"); } catch (SQLException e) { - context.verifyAuthzException(e); + Assert.fail("Exception not expected."); } // negative test: test user can't query column of view try { @@ -255,18 +256,18 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura context.verifyAuthzException(e); } - // negative test: test user can't query view + // positive test: test user can query column of view + // Hive 2 adds a new column view authorization as part of the Column Pruning feature + // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled try { statement.execute("SELECT COUNT(A) FROM VIEW_1"); - Assert.fail("Expected SQL exception"); } catch (SQLException e) { - context.verifyAuthzException(e); + Assert.fail("Exception not expected."); } try { statement.execute("SELECT COUNT(B) FROM VIEW_1"); - Assert.fail("Expected SQL exception"); } catch (SQLException e) { - context.verifyAuthzException(e); + Assert.fail("Exception not expected."); } // negative test: test user can't create a new view @@ -378,12 +379,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura Statement statement = context.createStatement(connection); statement.execute("USE DB_1"); - // test user can't execute query VIEW_1 JOIN VIEW_2 + // test user can execute query VIEW_1 JOIN VIEW_2 + // Hive 2 adds a new column view authorization as part of the Column Pruning feature + // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled try { statement.execute("SELECT COUNT(*) FROM VIEW_1 V1 JOIN VIEW_2 V2 ON (V1.B = V2.B)"); - Assert.fail("Expected SQL Exception"); } catch (SQLException e) { - context.verifyAuthzException(e); + Assert.fail("Exception not expected"); } // test user can't execute query VIEW_1 JOIN TAB_2 @@ -438,12 +440,13 @@ public class TestPrivilegesAtColumnScope extends AbstractTestWithStaticConfigura context.verifyAuthzException(e); } - // test user can't execute query VIEW_1 JOIN VIEW_2 + // test user can execute query VIEW_1 JOIN VIEW_2 + // Hive 2 adds a new column view authorization as part of the Column Pruning feature + // See Hive ColumnPrunerSelectProc.process() on how view authorization is handled try { statement.execute("SELECT COUNT(*) FROM VIEW_1 V1 JOIN VIEW_2 V2 ON (V1.B = V2.B)"); - Assert.fail("Expected SQL Exception"); } catch (SQLException e) { - context.verifyAuthzException(e); + Assert.fail("Exception not expected"); } // test user can't execute query TAB_1 JOIN TAB_2 http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java index d5eb137..7d41348 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java @@ -206,6 +206,16 @@ public class HiveServerFactory { String hadoopTempDir = System.getProperty("java.io.tmpdir") + File.separator + "hadoop-tmp"; properties.put("hadoop.tmp.dir", hadoopTempDir); + // This configuration will avoid that the HMS fails if the metastore schema has not version + // information. For some reason, HMS does not set a version initially on our tests. + properties.put(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false"); + + // Disable join cartesian checks to allow Sentry tests to pass + properties.put(ConfVars.HIVE_STRICT_CHECKS_CARTESIAN.varname, "false"); + + // Disable capability checks (these checks do not work when Hive is in testing mode) + properties.put(ConfVars.METASTORE_CAPABILITY_CHECK.varname, "false"); + if (!properties.containsKey(METASTORE_BYPASS)) { properties.put(METASTORE_BYPASS, "hive,impala," + System.getProperty("user.name", "")); } else { http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java index d0139ad..f14cbb6 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; @@ -138,7 +139,7 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends public void alterPartitionWithLocation(HiveMetaStoreClient client, Partition partition, String location) throws Exception { partition.getSd().setLocation(location); - client.alter_partition(partition.getDbName(), partition.getTableName(), partition); + client.alter_partition(partition.getDbName(), partition.getTableName(), partition, new EnvironmentContext()); } public void renamePartition(HiveMetaStoreClient client, Partition partition, Partition newPartition) throws Exception { http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java index 7f53527..f8f304f 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestMetastoreEndToEnd.java @@ -507,14 +507,14 @@ public class TestMetastoreEndToEnd extends client = context.getMetaStoreClient(USER1_1); Partition newPartition = client.getPartition(dbName, tabName1, partVals1); newPartition.getSd().setLocation(tabDir1); - client.alter_partition(dbName, tabName1, newPartition); + client.alter_partition(dbName, tabName1, newPartition, null); client.close(); // user with Table and URI privileges should be able to alter partition set location client = context.getMetaStoreClient(USER2_1); newPartition = client.getPartition(dbName, tabName1, partVals2); newPartition.getSd().setLocation(tabDir1); - client.alter_partition(dbName, tabName1, newPartition); + client.alter_partition(dbName, tabName1, newPartition, null); client.close(); policyFile.addRolesToGroup(USERGROUP3, db_all_role); @@ -524,7 +524,7 @@ public class TestMetastoreEndToEnd extends newPartition = client.getPartition(dbName, tabName1, partVals2); newPartition.getSd().setLocation(tabDir1); try { - client.alter_partition(dbName, tabName1, newPartition); + client.alter_partition(dbName, tabName1, newPartition, null); fail("alter partition with location should have failed"); } catch (MetaException e) { Context.verifyMetastoreAuthException(e); http://git-wip-us.apache.org/repos/asf/sentry/blob/5f64fe9f/sentry-tests/sentry-tests-solr/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/pom.xml b/sentry-tests/sentry-tests-solr/pom.xml index 723fa9d..9d400b2 100644 --- a/sentry-tests/sentry-tests-solr/pom.xml +++ b/sentry-tests/sentry-tests-solr/pom.xml @@ -110,6 +110,10 @@ limitations under the License. <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-shims</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -195,6 +199,10 @@ limitations under the License. <groupId>org.eclipse.jetty.aggregate</groupId> <artifactId>jetty-all</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-llap-server</artifactId> + </exclusion> </exclusions> </dependency> <dependency>
