Repository: incubator-sentry Updated Branches: refs/heads/master e0267df7c -> 184968e81
SENTRY-199: Create tool that will convert policy file into into DB store (Joey Echeverria via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/184968e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/184968e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/184968e8 Branch: refs/heads/master Commit: 184968e814ecdf1a161847f8b4831638fdbdb45b Parents: e0267df Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Jun 20 16:38:11 2014 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Jun 20 16:38:11 2014 -0700 ---------------------------------------------------------------------- sentry-binding/sentry-binding-hive/pom.xml | 2 - .../binding/hive/authz/SentryConfigTool.java | 140 +++++++++++++++- .../main/java/org/apache/sentry/SentryMain.java | 2 + .../file/SimpleFileProviderBackend.java | 4 + .../sentry/tests/e2e/hive/TestPolicyImport.java | 164 +++++++++++++++++++ 5 files changed, 309 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/184968e8/sentry-binding/sentry-binding-hive/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/pom.xml b/sentry-binding/sentry-binding-hive/pom.xml index 4974108..f2e1385 100644 --- a/sentry-binding/sentry-binding-hive/pom.xml +++ b/sentry-binding/sentry-binding-hive/pom.xml @@ -80,12 +80,10 @@ limitations under the License. <dependency> <groupId>org.apache.sentry</groupId> <artifactId>sentry-provider-file</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.apache.sentry</groupId> <artifactId>sentry-policy-db</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/184968e8/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryConfigTool.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryConfigTool.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryConfigTool.java index 370c00d..1c32946 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryConfigTool.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/SentryConfigTool.java @@ -17,6 +17,8 @@ package org.apache.sentry.binding.hive.authz; +import com.google.common.collect.Table; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; @@ -38,10 +40,20 @@ import org.apache.sentry.binding.hive.HiveAuthzBindingHook; import org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars; + import org.apache.sentry.core.common.SentryConfigurationException; import org.apache.sentry.core.common.Subject; +import org.apache.sentry.core.model.db.AccessConstants; +import org.apache.sentry.core.model.db.DBModelAuthorizable; import org.apache.sentry.core.model.db.Server; +import org.apache.sentry.policy.db.DBModelAuthorizables; import org.apache.sentry.provider.common.AuthorizationProvider; +import org.apache.sentry.provider.common.ProviderBackendContext; +import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; +import org.apache.sentry.provider.db.service.thrift.TSentryRole; +import org.apache.sentry.provider.file.KeyValue; +import org.apache.sentry.provider.file.PolicyFileConstants; +import org.apache.sentry.provider.file.SimpleFileProviderBackend; import java.security.CodeSource; import java.sql.Connection; @@ -49,6 +61,7 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.HashSet; import java.util.Set; public class SentryConfigTool { @@ -60,6 +73,7 @@ public class SentryConfigTool { private String passWord = null; private boolean listPrivs = false; private boolean validate = false; + private boolean importPolicy = false; private HiveConf hiveConf = null; private HiveAuthzConf authzConf = null; private AuthorizationProvider sentryProvider = null; @@ -100,6 +114,14 @@ public class SentryConfigTool { this.validate = validate; } + public boolean isImportPolicy() { + return importPolicy; + } + + public void setImportPolicy(boolean importPolicy) { + this.importPolicy = importPolicy; + } + public String getSentrySiteFile() { return sentrySiteFile; } @@ -229,6 +251,111 @@ public class SentryConfigTool { System.out.println("No errors found in the policy file"); } + // import policy files + public void importPolicy() throws Exception { + final String requestorUserName = "hive"; + SimpleFileProviderBackend policyFileBackend; + SentryPolicyServiceClient client; + + policyFileBackend = new SimpleFileProviderBackend(getAuthzConf(), + getAuthzConf().get(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar())); + ProviderBackendContext context = new ProviderBackendContext(); + context.setAllowPerDatabase(true); + policyFileBackend.initialize(context); + client = new SentryPolicyServiceClient(getAuthzConf()); + Set<String> roles = new HashSet<String>(); + for (TSentryRole sentryRole : client.listRoles(requestorUserName)) { + roles.add(sentryRole.getRoleName()); + } + + Table<String, String, Set<String>> groupRolePrivilegeTable = + policyFileBackend.getGroupRolePrivilegeTable(); + for(String groupName : groupRolePrivilegeTable.rowKeySet()) { + for(String roleName : groupRolePrivilegeTable.columnKeySet()) { + if (!roles.contains(roleName)) { + client.createRole(requestorUserName, roleName); + System.out.println(String.format("CREATE ROLE %s;", roleName)); + roles.add(roleName); + } + + Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName); + if (privileges == null) { + continue; + } + client.grantRoleToGroup(requestorUserName, groupName, roleName); + System.out.println(String.format("GRANT ROLE %s TO GROUP %s;", + roleName, groupName)); + + for (String permission : privileges) { + String server = null; + String database = null; + String table = null; + String uri = null; + String action = AccessConstants.ALL; + for (String authorizable : PolicyFileConstants.AUTHORIZABLE_SPLITTER. + trimResults().split(permission)) { + KeyValue kv = new KeyValue(authorizable); + DBModelAuthorizable a = DBModelAuthorizables.from(kv); + if (a == null) { + action = kv.getValue(); + continue; + } + + switch (a.getAuthzType()) { + case Server: + server = a.getName(); + break; + case Db: + database = a.getName(); + break; + case Table: + case View: + table = a.getName(); + break; + case URI: + uri = a.getName(); + break; + default: + break; + } + } + + if (uri != null) { + System.out.println(String.format( + "GRANT ALL ON URI %s TO ROLE %s; # server=%s", + uri, roleName, server)); + + client.grantURIPrivilege(requestorUserName, roleName, server, uri); + } else if (table != null && !AccessConstants.ALL.equals(table)) { + System.out.println(String.format( + "GRANT %s ON TABLE %s TO ROLE %s; # server=%s, database=%s", + "*".equals(action) ? "ALL" : action.toUpperCase(), table, + roleName, server, database)); + + client.grantTablePrivilege(requestorUserName, roleName, server, + database, table, action); + } else if (database != null && !AccessConstants.ALL.equals(database)) { + System.out.println(String.format( + "GRANT %s ON DATABASE %s TO ROLE %s; # server=%s", + "*".equals(action) ? "ALL" : action.toUpperCase(), + database, roleName, server)); + + client.grantDatabasePrivilege(requestorUserName, roleName, server, + database, action); + } else if (server != null) { + System.out.println(String.format("GRANT ALL ON SERVER %s TO ROLE %s;", + server, roleName)); + + client.grantServerPrivilege(requestorUserName, roleName, server); + } else { + System.out.println(String.format("No grant for permission %s", + permission)); + } + } + } + } + } + // list permissions for given user public void listPrivs() throws Exception { getSentryProvider().validateResource(true); @@ -363,7 +490,7 @@ public class SentryConfigTool { * <pre> * -d,--debug Enable debug output * -e,--query <arg> Query privilege verification, requires -u - * -h,--help Print usage + * -h,--help Print usage * -i,--policyIni <arg> Policy file path * -j,--jdbcURL <arg> JDBC URL * -l,--listPrivs,--listPerms List privilges for given user, requires -u @@ -371,6 +498,7 @@ public class SentryConfigTool { * -s,--sentry-site <arg> sentry-site file path * -u,--user <arg> user name * -v,--validate Validate policy file + * -I,--import Import policy file * </pre> * @param args */ @@ -397,6 +525,9 @@ public class SentryConfigTool { "list privileges for given user, requires -u"); listPrivsOpt.setRequired(false); + Option importOpt = new Option("I", "import", false, + "Import policy file"); + // required args OptionGroup sentryOptGroup = new OptionGroup(); sentryOptGroup.addOption(helpOpt); @@ -404,6 +535,7 @@ public class SentryConfigTool { sentryOptGroup.addOption(queryOpt); sentryOptGroup.addOption(listPermsOpt); sentryOptGroup.addOption(listPrivsOpt); + sentryOptGroup.addOption(importOpt); sentryOptGroup.setRequired(true); sentryOptions.addOptionGroup(sentryOptGroup); @@ -455,6 +587,8 @@ public class SentryConfigTool { setListPrivs(true); } else if (opt.getOpt().equals("v")) { setValidate(true); + } else if (opt.getOpt().equals("I")) { + setImportPolicy(true); } else if (opt.getOpt().equals("h")) { usage(sentryOptions); } else if (opt.getOpt().equals("d")) { @@ -495,6 +629,10 @@ public class SentryConfigTool { sentryTool.validatePolicy(); } + if (sentryTool.isImportPolicy()) { + sentryTool.importPolicy(); + } + // list permissions for give user if (sentryTool.isListPrivs()) { sentryTool.listPrivs(); http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/184968e8/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/SentryMain.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/SentryMain.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/SentryMain.java index 58233e0..497c8a4 100644 --- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/SentryMain.java +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/SentryMain.java @@ -32,6 +32,7 @@ public class SentryMain { private static final String HELP_SHORT = "h"; private static final String HELP_LONG = "help"; private static final String COMMAND = "command"; + private static final String HIVE_CONF = "hiveconf"; private static final String LOG4J_CONF = "log4jConf"; private static final ImmutableMap<String, String> COMMANDS = ImmutableMap .<String, String>builder() @@ -45,6 +46,7 @@ public class SentryMain { CommandLineParser parser = new GnuParser(); Options options = new Options(); options.addOption(HELP_SHORT, HELP_LONG, false, "Print this help text"); + options.addOption(HIVE_CONF, true, "Set hive configuration variables"); options.addOption(null, COMMAND, true, "Command to run. Options: " + COMMANDS.keySet()); options.addOption(null, LOG4J_CONF, true, "Location of log4j properties file"); //Ignore unrecognized options: service and config-tool options http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/184968e8/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java index 61eb694..fa5ab69 100644 --- a/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java +++ b/sentry-provider/sentry-provider-file/src/main/java/org/apache/sentry/provider/file/SimpleFileProviderBackend.java @@ -382,4 +382,8 @@ public class SimpleFileProviderBackend implements ProviderBackend { } } } + + public Table<String, String, Set<String>> getGroupRolePrivilegeTable() { + return groupRolePrivilegeTable; + } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/184968e8/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPolicyImport.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPolicyImport.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPolicyImport.java new file mode 100644 index 0000000..948b0c4 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPolicyImport.java @@ -0,0 +1,164 @@ +/* + * Copyright 2014 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.hive; + +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.apache.sentry.SentryUserException; +import org.apache.sentry.binding.hive.authz.SentryConfigTool; +import org.apache.sentry.core.model.db.AccessConstants; +import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient; +import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege; +import org.apache.sentry.provider.db.service.thrift.TSentryRole; +import org.apache.sentry.provider.file.PolicyFile; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestPolicyImport extends AbstractTestWithStaticConfiguration { + + private static String prefix; + private PolicyFile policyFile; + private SentryConfigTool configTool; + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception{ + useSentryService = true; + AbstractTestWithStaticConfiguration.setupTestStaticConfiguration(); + } + + @Before + public void setup() throws Exception { + policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); + policyFile.addGroupsToUser("hive", ADMINGROUP); + policyFile.addGroupsToUser(ADMIN1, ADMINGROUP); + + configTool = new SentryConfigTool(); + String hiveServer2 = System.getProperty("sentry.e2etest.hiveServer2Type", + "InternalHiveServer2"); + String policyOnHDFS = System.getProperty( + "sentry.e2etest.hive.policyOnHDFS", "true"); + if (policyOnHDFS.trim().equalsIgnoreCase("true") + && (hiveServer2.equals("UnmanagedHiveServer2"))) { + String policyLocation = System.getProperty( + "sentry.e2etest.hive.policy.location", "/user/hive/sentry"); + prefix = "hdfs://" + policyLocation + "/"; + } else { + prefix = "file://" + context.getPolicyFile().getParent() + "/"; + } + + } + + @Test + public void testImportPolicy() throws Exception { + policyFile.addRolesToGroup("analyst", "analyst_role", "customers_select_role"); + policyFile.addRolesToGroup("jranalyst", "junior_analyst_role"); + policyFile.addRolesToGroup("manager", "analyst_role", "junior_analyst_role", + "customers_insert_role", "customers_select_role"); + policyFile.addRolesToGroup("customers_admin", "customers_admin_role"); + + policyFile.addPermissionsToRole("analyst_role", "server=server1->db=analyst_db", + "server=server1->db=jranalyst_db->table=*->action=select"); + policyFile.addPermissionsToRole("junior_analyst_role", "server=server1->db=jranalyst_db"); + policyFile.addPermissionsToRole("customers_admin_role", "server=server1->db=customers"); + policyFile.addPermissionsToRole("customers_insert_role", "server=server1->db=customers->table=*->action=insert"); + policyFile.addPermissionsToRole("customers_select_role", "server=server1->db=customers->table=*->action=select"); + + policyFile.write(context.getPolicyFile()); + + configTool.setImportPolicy(true); + configTool.setPolicyFile(context.getPolicyFile().getPath()); + configTool.setupConfig(); + + configTool.importPolicy(); + + SentryPolicyServiceClient client = new SentryPolicyServiceClient(configTool.getAuthzConf()); + verifyRoles(client, "analyst", "analyst_role", "customers_select_role"); + verifyRoles(client, "jranalyst", "junior_analyst_role"); + verifyRoles(client, "manager", "analyst_role", "junior_analyst_role", + "customers_insert_role", "customers_select_role"); + verifyRoles(client, "customers_admin", "customers_admin_role"); + + verifyPrivileges(client, "analyst_role", + createPrivilege(AccessConstants.ALL, "analyst_db", null, null), + createPrivilege(AccessConstants.SELECT, "jranalyst_db", null, null)); + verifyPrivileges(client, "junior_analyst_role", + createPrivilege(AccessConstants.ALL, "jranalyst_db", null, null)); + verifyPrivileges(client, "customers_admin_role", + createPrivilege(AccessConstants.ALL, "customers", null, null)); + verifyPrivileges(client, "customers_insert_role", + createPrivilege(AccessConstants.INSERT, "customers", null, null)); + verifyPrivileges(client, "customers_select_role", + createPrivilege(AccessConstants.SELECT, "customers", null, null)); + } + + private void verifyRoles(SentryPolicyServiceClient client, String group, String ... roles) throws SentryUserException { + Set<String> expectedRoles = new HashSet<String>(Arrays.asList(roles)); + Set<String> actualRoles = new HashSet<String>(); + + Set<TSentryRole> groupRoles = client.listRolesByGroupName("hive", group); + for (TSentryRole role : groupRoles) { + actualRoles.add(role.getRoleName()); + } + + assertEquals("Expected roles don't match.", expectedRoles, actualRoles); + } + + private void verifyPrivileges(SentryPolicyServiceClient client, String role, TSentryPrivilege ... privileges) throws SentryUserException { + Set<TSentryPrivilege> expectedPrivileges = new HashSet<TSentryPrivilege>(Arrays.asList(privileges)); + Set<TSentryPrivilege> actualPrivileges = client.listAllPrivilegesByRoleName("hive", role); + for (TSentryPrivilege privilege : actualPrivileges) { + privilege.unsetCreateTime(); + privilege.unsetGrantorPrincipal(); + privilege.unsetPrivilegeName(); + } + + assertEquals("Expected privileges don't match.", expectedPrivileges, actualPrivileges); + } + + private TSentryPrivilege createPrivilege(String action, String dbName, String tableName, String uri) { + String scope = "SERVER"; + if (uri != null) { + scope = "URI"; + } else if (dbName != null) { + if (tableName != null) { + scope = "TABLE"; + } else { + scope = "DATABASE"; + } + } + + TSentryPrivilege privilege = new TSentryPrivilege(scope, "server1", action); + if (dbName != null) { + privilege.setDbName(dbName); + } + + if (tableName != null) { + privilege.setDbName(tableName); + } + + if (uri != null) { + privilege.setURI(uri); + } + + return privilege; + } + +}
