Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 78bd5ded7 -> d5176b2ea
SENTRY-1577: Support "create function using jar" for hive when Sentry is enabled (Aihua Xu, Reviewed by: Hao Hao, Alexander Kolbasov) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/d5176b2e Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/d5176b2e Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/d5176b2e Branch: refs/heads/sentry-ha-redesign Commit: d5176b2ea2e80d51f49d9e13075d794436fbe504 Parents: 78bd5de Author: Alexander Kolbasov <[email protected]> Authored: Tue Jan 24 22:23:19 2017 -0800 Committer: Alexander Kolbasov <[email protected]> Committed: Wed Jan 25 11:39:00 2017 -0800 ---------------------------------------------------------------------- .../binding/hive/HiveAuthzBindingHookBase.java | 38 ++++++++++-- .../hive/SentryOnFailureHookContext.java | 5 +- .../hive/SentryOnFailureHookContextImpl.java | 13 ++-- .../binding/hive/v2/HiveAuthzBindingHookV2.java | 14 ++++- .../binding/hive/HiveAuthzBindingHook.java | 14 ++++- sentry-tests/data/xudf.jar | Bin 0 -> 2430 bytes .../e2e/hive/TestPrivilegesAtFunctionScope.java | 60 +++++++++++++++++-- 7 files changed, 124 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java index dd16960..0661054 100644 --- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java +++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.sentry.binding.hive.authz.HiveAuthzBinding; import org.apache.sentry.binding.hive.authz.HiveAuthzPrivileges; @@ -76,6 +77,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerHook { @@ -85,7 +87,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH protected final HiveAuthzConf authzConf; protected Database currDB = Database.ALL; protected Table currTab; - protected AccessURI udfURI; + protected List<AccessURI> udfURIs; protected AccessURI serdeURI; protected AccessURI partitionURI; protected Table currOutTab = null; @@ -120,6 +122,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH throw new IllegalStateException("Session HiveConf is null"); } authzConf = loadAuthzConf(hiveConf); + udfURIs = Lists.newArrayList(); hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf); String serdeWhiteLists = authzConf.get(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST, @@ -174,7 +177,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH HiveOperation hiveOp, AuthorizationException e) { SentryOnFailureHookContext hookCtx = new SentryOnFailureHookContextImpl( context.getCommand(), context.getInputs(), context.getOutputs(), - hiveOp, currDB, currTab, udfURI, null, context.getUserName(), + hiveOp, currDB, currTab, udfURIs, null, context.getUserName(), context.getIpAddress(), e, context.getConf()); String csHooks = authzConf.get( HiveAuthzConf.AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), "").trim(); @@ -188,6 +191,33 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH } } + /** + * The command 'create function ... using jar <jar resources>' can create a function + * with the supplied jar resources in the command, which is translated into ASTNode being + * [functionName functionClass resourceList] and resourceList being [resourceType resourcePath]. + * This function collects all the jar paths for the supplied jar resources. + * + * @param ast the AST node for the command + * @return the jar path list if any or an empty list + */ + protected List<String> getFunctionJars(ASTNode ast) { + ASTNode resourcesNode = (ASTNode) ast.getFirstChildWithType(HiveParser.TOK_RESOURCE_LIST); + + List<String> resources = new ArrayList<String>(); + if (resourcesNode != null) { + for (int idx = 0; idx < resourcesNode.getChildCount(); ++idx) { + ASTNode resNode = (ASTNode) resourcesNode.getChild(idx); + ASTNode resTypeNode = (ASTNode) resNode.getChild(0); + ASTNode resUriNode = (ASTNode) resNode.getChild(1); + if (resTypeNode.getType() == HiveParser.TOK_JAR) { + resources.add(PlanUtils.stripQuotes(resUriNode.getText())); + } + } + } + + return resources; + } + @VisibleForTesting protected static AccessURI extractPartition(ASTNode ast) throws SemanticException { for (int i = 0; i < ast.getChildCount(); i++) { @@ -378,10 +408,10 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH * - CREATE TEMP FUNCTION * - DROP TEMP FUNCTION. */ - if (udfURI != null) { + if (!udfURIs.isEmpty()) { List<DBModelAuthorizable> udfUriHierarchy = new ArrayList<DBModelAuthorizable>(); udfUriHierarchy.add(hiveAuthzBinding.getAuthServer()); - udfUriHierarchy.add(udfURI); + udfUriHierarchy.addAll(udfURIs); inputHierarchy.add(udfUriHierarchy); for (WriteEntity writeEntity : outputs) { List<DBModelAuthorizable> entityHierarchy = new ArrayList<DBModelAuthorizable>(); http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java index c101a4f..ac14fb1 100644 --- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java +++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java @@ -18,6 +18,7 @@ package org.apache.sentry.binding.hive; +import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -76,9 +77,9 @@ public interface SentryOnFailureHookContext { Table getTable(); /** - * @return the udf URI + * @return the udf URIs */ - AccessURI getUdfURI(); + List<AccessURI> getUdfURIs(); /** * @return the partition URI http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java index f97d7f3..15bfbea 100644 --- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java +++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java @@ -18,6 +18,7 @@ package org.apache.sentry.binding.hive; +import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -39,14 +40,14 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex private final String ipAddress; private final Database database; private final Table table; - private final AccessURI udfURI; + private final List<AccessURI> udfURIs; private final AccessURI partitionURI; private final AuthorizationException authException; private final Configuration conf; public SentryOnFailureHookContextImpl(String command, Set<ReadEntity> inputs, Set<WriteEntity> outputs, HiveOperation hiveOp, - Database db, Table tab, AccessURI udfURI, AccessURI partitionURI, + Database db, Table tab, List<AccessURI> udfURIs, AccessURI partitionURI, String userName, String ipAddress, AuthorizationException e, Configuration conf) { this.command = command; @@ -57,7 +58,7 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex this.ipAddress = ipAddress; this.database = db; this.table = tab; - this.udfURI = udfURI; + this.udfURIs = udfURIs; this.partitionURI = partitionURI; this.authException = e; this.conf = conf; @@ -104,8 +105,8 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex } @Override - public AccessURI getUdfURI() { - return udfURI; + public List<AccessURI> getUdfURIs() { + return udfURIs; } @Override @@ -122,4 +123,4 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex public Configuration getConf() { return conf; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java b/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java index 91a2507..018ebf3 100644 --- a/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java +++ b/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java @@ -69,10 +69,20 @@ public class HiveAuthzBindingHookV2 extends HiveAuthzBindingHookBase { throw new SemanticException("Could not find the jar for UDF class " + udfClassName + "to validate privileges"); } - udfURI = parseURI(udfSrc.getLocation().toString(), true); + udfURIs.add(parseURI(udfSrc.getLocation().toString(), true)); } catch (ClassNotFoundException e) { - throw new SemanticException("Error retrieving udf class", e); + List<String> functionJars = getFunctionJars(ast); + if (functionJars.isEmpty()) { + throw new SemanticException("Error retrieving udf class", e); + } else { + // Add the jars from the command "Create function using jar" to the access list + // Defer to hive to check if the class is in the jars + for(String jar : functionJars) { + udfURIs.add(parseURI(jar, false)); + } + } } + // create/drop function is allowed with any database currDB = Database.ALL; break; http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java index 7242fde..c7afe1b 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java @@ -208,10 +208,20 @@ public class HiveAuthzBindingHook extends HiveAuthzBindingHookBase { throw new SemanticException("Could not find the jar for UDF class " + udfClassName + "to validate privileges"); } - udfURI = parseURI(udfSrc.getLocation().toString(), true); + udfURIs.add(parseURI(udfSrc.getLocation().toString(), true)); } catch (ClassNotFoundException e) { - throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e); + List<String> functionJars = getFunctionJars(ast); + if (functionJars.isEmpty()) { + throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e); + } else { + // Add the jars from the command "Create function using jar" to the access list + // Defer to hive to check if the class is in the jars + for(String jar : functionJars) { + udfURIs.add(parseURI(jar, false)); + } + } } + // create/drop function is allowed with any database currDB = Database.ALL; break; http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-tests/data/xudf.jar ---------------------------------------------------------------------- diff --git a/sentry-tests/data/xudf.jar b/sentry-tests/data/xudf.jar new file mode 100644 index 0000000..5bc4cee Binary files /dev/null and b/sentry-tests/data/xudf.jar differ http://git-wip-us.apache.org/repos/asf/sentry/blob/d5176b2e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java index 61fecc7..e18da1f 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java @@ -17,9 +17,7 @@ printf_test_3 * Licensed to the Apache Software Foundation (ASF) under one or mo package org.apache.sentry.tests.e2e.hive; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; import java.io.File; import java.io.FileOutputStream; @@ -86,6 +84,8 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu String udf1ClassName = "org.apache.sentry.tests.e2e.hive.TestUDF"; CodeSource udf1Src = Class.forName(udf1ClassName).getProtectionDomain().getCodeSource(); String udf1Location = udf1Src.getLocation().getPath(); + String udf2Location = new File("../data/xudf.jar").getCanonicalPath(); + Connection connection = context.createConnection(ADMIN1); Statement statement = context.createStatement(connection); statement.execute("DROP DATABASE IF EXISTS " + DB1 + " CASCADE"); @@ -99,13 +99,14 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu context.close(); policyFile - .addRolesToGroup(USERGROUP1, "db1_all", "UDF1_JAR", "UDF_JAR", "data_read") + .addRolesToGroup(USERGROUP1, "db1_all", "UDF2_JAR", "UDF1_JAR", "UDF_JAR", "data_read") .addRolesToGroup(USERGROUP2, "db1_tab1", "UDF_JAR") .addRolesToGroup(USERGROUP3, "db1_tab1") .addPermissionsToRole("db1_all", "server=server1->db=" + DB1) .addPermissionsToRole("db1_tab1", "server=server1->db=" + DB1 + "->table=" + tableName1) .addPermissionsToRole("UDF_JAR", "server=server1->uri=file://" + udfLocation) .addPermissionsToRole("UDF1_JAR", "server=server1->uri=file://" + udf1Location) + .addPermissionsToRole("UDF2_JAR", "server=server1->uri=file://" + udf2Location) .addPermissionsToRole("data_read", "server=server1->URI=" + "file:///tmp"); writePolicyFile(policyFile); } @@ -266,6 +267,57 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu context.close(); } + /** + * Test create function using jar functionality + * @throws Exception + */ + @Test + public void testAndVerifyFuncPrivilegesPart4() throws Exception { + setUpContext(); + Connection connection = context.createConnection(USER1_1); + Statement statement = context.createStatement(connection); + statement.execute("USE " + DB1); + + // USER1 has URI perm on jarFiles + try { + String udfLocation = new File("../data/xudf.jar").getCanonicalPath(); + + statement + .execute("CREATE FUNCTION " + + DB1 + + ".xadd AS 'xudf.XAdd'" + + " using jar 'file://" + udfLocation + "'"); + ResultSet rs = statement.executeQuery("select xadd(1)"); + assertTrue(rs.next()); + assertTrue(rs.getInt(1) == 1); + } catch (SQLException e) { + assertFalse("CREATE FUNCTION should succeed for user1:" + e, true); + } finally { + connection.close(); + } + + connection = context.createConnection(USER2_1); + statement = context.createStatement(connection); + statement.execute("USE " + DB1); + + // USER2 doesn't have URI perm on jarFiles + try { + String udfLocation = new File("../data/xudf.jar").getCanonicalPath(); + + statement + .execute("CREATE FUNCTION " + + DB1 + + ".xadd AS 'xudf.XAdd'" + + " using jar 'file://" + udfLocation + "'"); + assertFalse("CREATE FUNCTION should fail for user2", true); + } catch (SQLException e) { + context.verifyAuthzException(e); + } finally { + connection.close(); + } + context.close(); + } + @Test public void testUdfWhiteList () throws Exception { String tableName1 = "tab1";
