Repository: incubator-sentry Updated Branches: refs/heads/upstream-master [created] 34b5afc36
SENTRY-1087:Capture URI when using Hive Serdes (Hao Hao, Reviewed by: Sravya Tirukkovalur and Lenni Kuff) Change-Id: I06d19ffc8e5dfc8fd16c6c5a79ea40270580fb72 Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/34b5afc3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/34b5afc3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/34b5afc3 Branch: refs/heads/upstream-master Commit: 34b5afc36feaaea19ea129e985c0cad95629c4e5 Parents: f727a0c Author: hahao <hao....@cloudera.com> Authored: Tue Mar 1 17:18:29 2016 -0800 Committer: hahao <hao....@cloudera.com> Committed: Tue Mar 1 17:51:39 2016 -0800 ---------------------------------------------------------------------- .../binding/hive/HiveAuthzBindingHook.java | 92 ++++++++++++++- .../hive/authz/HiveAuthzPrivilegesMap.java | 3 +- .../sentry/binding/hive/conf/HiveAuthzConf.java | 8 ++ .../e2e/hive/TestCustomSerdePrivileges.java | 115 +++++++++++++++++++ .../e2e/hive/TestPrivilegesAtFunctionScope.java | 34 ++++++ 5 files changed, 250 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java index 08c0e98..dd33d2d 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Set; +import java.util.Arrays; import com.google.common.base.Preconditions; import org.apache.hadoop.hive.common.JavaUtils; @@ -44,6 +45,7 @@ import org.apache.hadoop.hive.ql.hooks.Entity.Type; import org.apache.hadoop.hive.ql.hooks.Hook; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.AbstractSemanticAnalyzerHook; @@ -87,9 +89,12 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { private Database currDB = Database.ALL; private Table currTab; private AccessURI udfURI; + private AccessURI serdeURI; private AccessURI partitionURI; private Table currOutTab = null; private Database currOutDB = null; + private final List<String> serdeWhiteList; + private boolean serdeURIPrivilegesEnabled; // True if this is a basic DESCRIBE <table> operation. False for other DESCRIBE variants // like DESCRIBE [FORMATTED|EXTENDED]. Required because Hive treats these stmts as the same @@ -113,6 +118,12 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { authzConf = loadAuthzConf(hiveConf); hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf); + String serdeWhiteLists = authzConf.get(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST, + HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST_DEFAULT); + serdeWhiteList = Arrays.asList(serdeWhiteLists.split(",")); + serdeURIPrivilegesEnabled = authzConf.getBoolean(HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED, + HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT); + FunctionRegistry.setupPermissionsForBuiltinUDFs("", HiveAuthzConf.HIVE_UDF_BLACK_LIST); } @@ -164,6 +175,16 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { currDB = new Database(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(0).getText())); break; case HiveParser.TOK_CREATETABLE: + + for (Node childNode : ast.getChildren()) { + ASTNode childASTNode = (ASTNode) childNode; + if ("TOK_TABLESERIALIZER".equals(childASTNode.getText())) { + ASTNode serdeNode = (ASTNode)childASTNode.getChild(0); + String serdeClassName = BaseSemanticAnalyzer.unescapeSQLString(serdeNode.getChild(0).getText()); + setSerdeURI(serdeClassName); + } + } + case HiveParser.TOK_CREATEVIEW: /* * Compiler doesn't create read/write entities for create table. @@ -283,7 +304,18 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { currOutDB = extractDatabase((ASTNode) ast.getChild(0)); currOutTab = extractTable((ASTNode) ast.getChild(0).getChild(0).getChild(0)); break; - default: + case HiveParser.TOK_ALTERTABLE: + + for (Node childNode : ast.getChildren()) { + ASTNode childASTNode = (ASTNode) childNode; + if ("TOK_ALTERTABLE_SERIALIZER".equals(childASTNode.getText())) { + ASTNode serdeNode = (ASTNode)childASTNode.getChild(0); + String serdeClassName = BaseSemanticAnalyzer.unescapeSQLString(serdeNode.getText()); + setSerdeURI(serdeClassName); + } + } + + default: currDB = getCanonicalDb(); break; } @@ -497,6 +529,13 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { outputHierarchy.add(dbHierarchy); getInputHierarchyFromInputs(inputHierarchy, inputs); + + if (serdeURI != null) { + List<DBModelAuthorizable> serdeUriHierarchy = new ArrayList<DBModelAuthorizable>(); + serdeUriHierarchy.add(hiveAuthzBinding.getAuthServer()); + serdeUriHierarchy.add(serdeURI); + outputHierarchy.add(serdeUriHierarchy); + } break; case TABLE: // workaround for add partitions @@ -535,6 +574,14 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { externalAuthorizableHierarchy.add(currOutTab); outputHierarchy.add(externalAuthorizableHierarchy); } + + if (serdeURI != null) { + List<DBModelAuthorizable> serdeUriHierarchy = new ArrayList<DBModelAuthorizable>(); + serdeUriHierarchy.add(hiveAuthzBinding.getAuthServer()); + serdeUriHierarchy.add(serdeURI); + outputHierarchy.add(serdeUriHierarchy); + } + break; case FUNCTION: /* The 'FUNCTION' privilege scope currently used for @@ -956,4 +1003,47 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook { throw new SemanticException(e); } } + + private static boolean hasPrefixMatch(List<String> prefixList, final String str) { + for (String prefix : prefixList) { + if (str.startsWith(prefix)) { + return true; + } + } + + return false; + } + + /** + * Set the Serde URI privileges. If the URI privileges are not set, which serdeURI will be null, + * the URI authorization checks will be skipped. + */ + private void setSerdeURI(String serdeClassName) throws SemanticException { + if (!serdeURIPrivilegesEnabled) { + return; + } + + // WhiteList Serde Jar can be used by any users. WhiteList checking is + // done by comparing the Java package name. The assumption is cluster + // admin will ensure there is no Java namespace collision. + // e.g org.apache.hadoop.hive.serde2 is used by hive and cluster admin should + // ensure no custom Serde class is introduced under the same namespace. + if (!hasPrefixMatch(serdeWhiteList, serdeClassName)) { + try { + CodeSource serdeSrc = Class.forName(serdeClassName, true, Utilities.getSessionSpecifiedClassLoader()).getProtectionDomain().getCodeSource(); + if (serdeSrc == null) { + throw new SemanticException("Could not resolve the jar for Serde class " + serdeClassName); + } + + String serdeJar = serdeSrc.getLocation().getPath(); + if (serdeJar == null || serdeJar.isEmpty()) { + throw new SemanticException("Could not find the jar for Serde class " + serdeClassName + "to validate privileges"); + } + + serdeURI = parseURI(serdeSrc.getLocation().toString(), true); + } catch (ClassNotFoundException e) { + throw new SemanticException("Error retrieving Serde class:" + e.getMessage(), e); + } + } + } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java index 0c3bee3..8e70492 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java @@ -39,6 +39,7 @@ public class HiveAuthzPrivilegesMap { HiveAuthzPrivileges tableCreatePrivilege = new HiveAuthzPrivileges.AuthzPrivilegeBuilder(). addOutputObjectPriviledge(AuthorizableType.Db, EnumSet.of(DBModelAction.CREATE)). addInputObjectPriviledge(AuthorizableType.URI, EnumSet.of(DBModelAction.ALL)).//TODO: make it optional + addOutputObjectPriviledge(AuthorizableType.URI, EnumSet.of(DBModelAction.ALL)). setOperationScope(HiveOperationScope.DATABASE). setOperationType(HiveOperationType.DDL). build(); @@ -225,7 +226,6 @@ public class HiveAuthzPrivilegesMap { hiveAuthzStmtPrivMap.put(HiveOperation.ALTERPARTITION_PROTECTMODE, alterTablePrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERPARTITION_SERDEPROPERTIES, alterTablePrivilege); - hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_SERIALIZER, alterTablePrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_MERGEFILES, alterTablePrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_SKEWED, alterTablePrivilege); @@ -238,6 +238,7 @@ public class HiveAuthzPrivilegesMap { hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_ADDPARTS, addPartitionPrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_RENAME, alterTableRenamePrivilege); + hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_SERIALIZER, alterTableAndUriPrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_LOCATION, alterTableAndUriPrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERPARTITION_LOCATION, alterTableAndUriPrivilege); hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTBLPART_SKEWED_LOCATION, alterTableAndUriPrivilege);//TODO: Needs test case http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java index 6b79dda..1093a09 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java @@ -51,6 +51,13 @@ public class HiveAuthzConf extends Configuration { public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT = "set,reset,reload"; + public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist"; + public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2"; + + // Disable the serde Uri privileges by default for backward compatibilities. + public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges"; + public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false; + public static final String HIVE_UDF_WHITE_LIST = "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," + "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," + @@ -76,6 +83,7 @@ public class HiveAuthzConf extends Configuration { "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath"; public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method"; + /** * Config setting definitions */ http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java new file mode 100644 index 0000000..6dfdb3c --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.hive; + +import com.google.common.collect.Maps; +import org.apache.sentry.binding.hive.conf.HiveAuthzConf; +import org.apache.sentry.provider.file.PolicyFile; +import org.junit.*; +import org.junit.Assert; +import org.junit.Test; + +import java.security.CodeSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; + +public class TestCustomSerdePrivileges extends AbstractTestWithHiveServer { + private static Context context; + private static Map<String, String> properties; + private PolicyFile policyFile; + + @BeforeClass + public static void setUp() throws Exception { + properties = Maps.newHashMap(); + + // Start the Hive Server without buildin Serde, such as + // "org.apache.hadoop.hive.serde2.OpenCSVSerde". Instead, + // used a bogus class name for testing. + properties.put(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST, "org.example.com"); + properties.put(HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED, "true"); + context = createContext(properties); + } + + @AfterClass + public static void tearDown() throws Exception { + if(context != null) { + context.close(); + } + } + + @Before + public void setupPolicyFile() throws Exception { + policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); + } + + /** + * User with db level access and Uri privileges on the Serde Jar should be able + * to create tables with Serde. + * User with db level access but without Uri privileges on the Serde Jar will fail + * on creating tables with Serde. + */ + @Test + public void testSerdePrivilegesWithoutBuildinJar() throws Exception { + String db = "db1"; + String tableName1 = "tab1"; + + String serdeClassName = "org.apache.hadoop.hive.serde2.OpenCSVSerde"; + CodeSource serdeSrc = Class.forName(serdeClassName).getProtectionDomain().getCodeSource(); + String serdeLocation = serdeSrc.getLocation().getPath(); + + policyFile + .addRolesToGroup(USERGROUP1, "db1_all") + .addRolesToGroup(USERGROUP2, "db1_all", "SERDE_JAR") + .addPermissionsToRole("db1_all", "server=server1->db=" + db) + .addPermissionsToRole("db1_tab1", "server=server1->db=" + db + "->table=" + tableName1) + .addPermissionsToRole("SERDE_JAR", "server=server1->uri=file://" + serdeLocation) + .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + policyFile.write(context.getPolicyFile()); + + Connection connection = context.createConnection(ADMIN1); + Statement statement = context.createStatement(connection); + statement.execute("DROP DATABASE IF EXISTS " + db + " CASCADE"); + statement.execute("CREATE DATABASE " + db); + context.close(); + + // User1 does not have the URI privileges to use the Serde Jar. + // The table creation will fail. + connection = context.createConnection(USER1_1); + statement = context.createStatement(connection); + statement.execute("USE " + db); + try { + statement.execute("create table " + db + "." + tableName1 + " (a string, b string) " + + "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' " + " STORED AS TEXTFILE"); + Assert.fail("Expect create table with Serde to fail"); + } catch (SQLException e) { + context.verifyAuthzException(e); + } + context.close(); + + // User2 has the URI privileges to use the Serde Jar. + // The table creation will succeed. + connection = context.createConnection(USER2_1); + statement = context.createStatement(connection); + statement.execute("USE " + db); + statement.execute("create table " + db + "." + tableName1 + " (a string, b string) ROW FORMAT" + + " SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' " + " STORED AS TEXTFILE"); + context.close(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java index cfaf7c3..bb8d61d 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java @@ -225,4 +225,38 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu statement.close(); connection.close(); } + + /** + * User with db level access should be able to create/alter tables with buildin Serde. + */ + @Test + public void testSerdePrivileges() throws Exception { + String tableName1 = "tab1"; + String tableName2 = "tab2"; + + Connection connection = context.createConnection(ADMIN1); + Statement statement = context.createStatement(connection); + statement.execute("DROP DATABASE IF EXISTS " + DB1 + " CASCADE"); + statement.execute("CREATE DATABASE " + DB1); + + context.close(); + + policyFile + .addRolesToGroup(USERGROUP1, "db1_all") + .addPermissionsToRole("db1_all", "server=server1->db=" + DB1); + writePolicyFile(policyFile); + + connection = context.createConnection(USER1_1); + statement = context.createStatement(connection); + statement.execute("USE " + DB1); + statement.execute("create table " + DB1 + "." + tableName1 + + " (a string, b string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' " + + " STORED AS TEXTFILE"); + + statement.execute("create table " + DB1 + "." + tableName2 + " (a string, b string)"); + statement.execute("alter table " + DB1 + "." + tableName2 + + " SET SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'"); + + context.close(); + } }
