http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini b/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini deleted file mode 100644 index c1bfe4b..0000000 --- a/sentry-provider/sentry-provider-db/src/test/resources/indexer_config_import_tool.ini +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[groups] -corporal = corporal_role -sergeant = corporal_role, sergeant_role -general = corporal_role, sergeant_role, general_role -commander_in_chief = corporal_role, sergeant_role, general_role, commander_in_chief_role - -[roles] -corporal_role = indexer=info->action=read, \ - indexer=info->action=write -sergeant_role = indexer=info->action=write -general_role = indexer=info->action=* -commander_in_chief_role = indexer=*
http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini b/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini deleted file mode 100644 index 03083a7..0000000 --- a/sentry-provider/sentry-provider-db/src/test/resources/indexer_invalid.ini +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[groups] - -[roles] - http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini deleted file mode 100644 index fbbebfc..0000000 --- a/sentry-provider/sentry-provider-db/src/test/resources/solr_case.ini +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[groups] -groupa = RoLe1 -groupb = rOlE1 -groupc = ROLE2 - -[roles] -RoLe1 = collection=* -rOlE1 = collection=* -ROLE2 = collection=* http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini deleted file mode 100644 index da7df4c..0000000 --- a/sentry-provider/sentry-provider-db/src/test/resources/solr_config_import_tool.ini +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[groups] -corporal = corporal_role -sergeant = corporal_role, sergeant_role -general = corporal_role, sergeant_role, general_role -commander_in_chief = corporal_role, sergeant_role, general_role, commander_in_chief_role - -[roles] -corporal_role = collection=info->action=query, \ - collection=info->action=update -sergeant_role = collection=info->action=update -general_role = collection=info->action=* -commander_in_chief_role = collection=* http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini b/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini deleted file mode 100644 index 03083a7..0000000 --- a/sentry-provider/sentry-provider-db/src/test/resources/solr_invalid.ini +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[groups] - -[roles] - http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-tools/pom.xml b/sentry-tools/pom.xml index 4d8fc89..b882c6f 100644 --- a/sentry-tools/pom.xml +++ b/sentry-tools/pom.xml @@ -26,6 +26,7 @@ limitations under the License. <modelVersion>4.0.0</modelVersion> <artifactId>sentry-tools</artifactId> + <version>2.1.0-SNAPSHOT</version> <dependencies> <dependency> @@ -33,27 +34,56 @@ limitations under the License. <artifactId>sentry-binding-hive</artifactId> </dependency> <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-provider-db</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-provider-db</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> </dependency> <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </dependency> + <dependency> <groupId>com.budhash.cliche</groupId> <artifactId>cliche-shell</artifactId> <version>0.9.3</version> </dependency> <dependency> - <groupId>org.apache.sentry</groupId> - <artifactId>sentry-provider-db</artifactId> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minikdc</artifactId> + <scope>test</scope> </dependency> </dependencies> <build> - <sourceDirectory>${basedir}/src/main/java</sourceDirectory> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <reuseForks>false</reuseForks> + </configuration> + </plugin> + </plugins> + <testResources> + <testResource> + <directory>src/test/resources</directory> + </testResource> + </testResources> </build> - - </project> http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java b/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java index e92155c..495d4cd 100644 --- a/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java +++ b/sentry-tools/src/main/java/org/apache/sentry/SentryMain.java @@ -24,7 +24,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.log4j.PropertyConfigurator; import org.apache.sentry.binding.hive.authz.SentryConfigTool; -import org.apache.sentry.provider.db.tools.SentrySchemaTool; +import org.apache.sentry.cli.tools.SentrySchemaTool; import org.apache.sentry.service.thrift.SentryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java new file mode 100644 index 0000000..8b5130c --- /dev/null +++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolCommon.java @@ -0,0 +1,349 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.cli.tools; + +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.Parser; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.sentry.core.common.ActiveRoleSet; +import org.apache.sentry.core.common.utils.PolicyFileConstants; +import org.apache.sentry.core.common.utils.PolicyFiles; +import org.apache.sentry.core.common.utils.Version; +import org.apache.sentry.policy.common.PrivilegeUtils; +import org.apache.sentry.provider.common.ProviderBackendContext; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory; +import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege; +import org.apache.sentry.provider.db.generic.service.thrift.TSentryRole; +import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter; +import org.apache.sentry.provider.file.SimpleFileProviderBackend; +import org.apache.shiro.config.Ini; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Sets; +import com.google.common.collect.Table; + +/** + * This class provides basic framework required to migrate permissions between different Sentry + * versions. Individual components (e.g. SOLR, KAFKA) needs to override the this class + * to provide component specific migration functionality. + */ +public abstract class PermissionsMigrationToolCommon { + private static final Logger LOGGER = LoggerFactory.getLogger(PermissionsMigrationToolCommon.class); + public static final String SOLR_SERVICE_NAME = "sentry.service.client.solr.service.name"; + + private Version sourceVersion; + private Optional<String> confPath = Optional.empty(); + private Optional<String> policyFile = Optional.empty(); + private Optional<String> outputFile = Optional.empty(); + private boolean dryRun = false; + + /** + * @return version of Sentry for which the privileges need to be migrated. + */ + public final Version getSourceVersion() { + return sourceVersion; + } + + /** + * This method returns the name of the component for the migration purpose. + * @param conf The Sentry configuration + * @return the name of the component + */ + protected abstract String getComponent(Configuration conf); + + + /** + * This method returns the name of the service name for the migration purpose. + * + * @param conf The Sentry configuration + * @return the name of the service + */ + protected abstract String getServiceName(Configuration conf); + + /** + * Migrate the privileges specified via <code>privileges</code>. + * + * @param privileges A collection of privileges to be migrated. + * @return A collection of migrated privileges + * An empty collection if migration is not necessary for the specified privileges. + */ + protected abstract Collection<String> transformPrivileges (Collection<String> privileges); + + /** + * parse arguments + * <pre> + * -s,--source Sentry source version + * -c,--sentry_conf <filepath> sentry config file path + * -p --policy_file <filepath> sentry (source) policy file path + * -o --output <filepath> sentry (target) policy file path + * -d --dry_run provides the output the migration for inspection without + * making any configuration changes. + * -h,--help print usage + * </pre> + * @param args + */ + protected boolean parseArgs(String [] args) { + Options options = new Options(); + + Option sourceVersionOpt = new Option("s", "source", true, "Source Sentry version"); + sourceVersionOpt.setRequired(true); + options.addOption(sourceVersionOpt); + + Option sentryConfPathOpt = new Option("c", "sentry_conf", true, + "sentry-site.xml file path (only required in case of Sentry service)"); + sentryConfPathOpt.setRequired(false); + options.addOption(sentryConfPathOpt); + + Option sentryPolicyFileOpt = new Option("p", "policy_file", true, + "sentry (source) policy file path (only in case of file based Sentry configuration)"); + sentryPolicyFileOpt.setRequired(false); + options.addOption(sentryPolicyFileOpt); + + Option sentryOutputFileOpt = new Option("o", "output", true, + "sentry (target) policy file path (only in case of file based Sentry configuration)"); + sentryOutputFileOpt.setRequired(false); + options.addOption(sentryOutputFileOpt); + + Option dryRunOpt = new Option("d", "dry_run", false, + "provides the output the migration for inspection without making actual configuration changes"); + dryRunOpt.setRequired(false); + options.addOption(dryRunOpt); + + // help option + Option helpOpt = new Option("h", "help", false, "Shell usage"); + helpOpt.setRequired(false); + options.addOption(helpOpt); + + // this Option is parsed first for help option + Options helpOptions = new Options(); + helpOptions.addOption(helpOpt); + + try { + Parser parser = new GnuParser(); + + // parse help option first + CommandLine cmd = parser.parse(helpOptions, args, true); + for (Option opt : cmd.getOptions()) { + if (opt.getOpt().equals("h")) { + // get the help option, print the usage and exit + usage(options); + return false; + } + } + + // without help option + cmd = parser.parse(options, args); + + String sourceVersionStr = null; + + for (Option opt : cmd.getOptions()) { + if (opt.getOpt().equals("s")) { + sourceVersionStr = opt.getValue(); + } else if (opt.getOpt().equals("c")) { + confPath = Optional.of(opt.getValue()); + } else if (opt.getOpt().equals("p")) { + policyFile = Optional.of(opt.getValue()); + } else if (opt.getOpt().equals("o")) { + outputFile = Optional.of(opt.getValue()); + } else if (opt.getOpt().equals("d")) { + dryRun = true; + } + } + + sourceVersion = Version.parse(sourceVersionStr); + + if (!(confPath.isPresent() || policyFile.isPresent())) { + System.out.println("Please select either file-based Sentry configuration (-p and -o flags)" + + " or Sentry service (-c flag) for migration."); + usage(options); + return false; + } + + if (confPath.isPresent() && (policyFile.isPresent() || outputFile.isPresent())) { + System.out.println("In order to migrate service based Sentry configuration," + + " do not specify either -p or -o parameters"); + usage(options); + return false; + } + + if (!confPath.isPresent() && (policyFile.isPresent() ^ outputFile.isPresent())) { + System.out.println("In order to migrate file based Sentry configuration," + + " please make sure to specify both -p and -o parameters."); + usage(options); + return false; + } + + } catch (ParseException | java.text.ParseException pe) { + System.out.println(pe.getMessage()); + usage(options); + return false; + } + return true; + } + + // print usage + private void usage(Options sentryOptions) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("sentryMigrationTool", sentryOptions); + } + + public void run() throws Exception { + if (policyFile.isPresent()) { + migratePolicyFile(); + } else { + migrateSentryServiceConfig(); + } + } + + private void migrateSentryServiceConfig() throws Exception { + Configuration conf = getSentryConf(); + String component = getComponent(conf); + String serviceName = getServiceName(conf); + GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, serviceName, false); + + // instantiate a client for sentry service. This sets the ugi, so must + // be done before getting the ugi below. + try(SentryGenericServiceClient client = + SentryGenericServiceClientFactory.create(getSentryConf())) { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + String requestorName = ugi.getShortUserName(); + + for (TSentryRole r : client.listAllRoles(requestorName, component)) { + for (TSentryPrivilege p : client.listAllPrivilegesByRoleName(requestorName, + r.getRoleName(), component, serviceName)) { + + String privilegeStr = converter.toString(p); + Collection<String> privileges = Collections.singleton(privilegeStr); + Collection<String> migrated = transformPrivileges(privileges); + if (!migrated.isEmpty()) { + LOGGER.info("{} For role {} migrating privileges from {} to {}", getDryRunMessage(), r.getRoleName(), + privileges, migrated); + + /* + * Note that it is not possible to provide transactional (all-or-nothing) behavior for these configuration + * changes since the Sentry client/server protocol does not support. e.g. under certain failure conditions + * like crash of Sentry server or network disconnect between client/server, it is possible that the migration + * can not complete but can also not be rolled back. Hence this migration tool relies on the fact that privilege + * grant/revoke operations are idempotent and hence re-execution of the migration tool will fix any inconsistency + * due to such failures. + **/ + boolean originalPermPresent = false; + for (String perm : migrated) { + if (perm.equalsIgnoreCase(privilegeStr)) { + originalPermPresent = true; + continue; + } + TSentryPrivilege x = converter.fromString(perm); + LOGGER.info("{} GRANT permission {}", getDryRunMessage(), perm); + if (!dryRun) { + client.grantPrivilege(requestorName, r.getRoleName(), component, x); + } + } + + // Revoke old permission (only if not part of migrated permissions) + if (!originalPermPresent) { + LOGGER.info("{} REVOKE permission {}", getDryRunMessage(), privilegeStr); + if (!dryRun) { + client.revokePrivilege(requestorName, r.getRoleName(), component, p); + } + } + } + } + } + } + } + + private void migratePolicyFile () throws Exception { + Configuration conf = getSentryConf(); + Path sourceFile = new Path (policyFile.get()); + SimpleFileProviderBackend policyFileBackend = new SimpleFileProviderBackend(conf, sourceFile); + ProviderBackendContext ctx = new ProviderBackendContext(); + policyFileBackend.initialize(ctx); + + Set<String> roles = Sets.newHashSet(); + Table<String, String, Set<String>> groupRolePrivilegeTable = + policyFileBackend.getGroupRolePrivilegeTable(); + + Ini output = PolicyFiles.loadFromPath(sourceFile.getFileSystem(conf), sourceFile); + Ini.Section rolesSection = output.get(PolicyFileConstants.ROLES); + + for (String groupName : groupRolePrivilegeTable.rowKeySet()) { + for (String roleName : policyFileBackend.getRoles(Collections.singleton(groupName), ActiveRoleSet.ALL)) { + if (!roles.contains(roleName)) { + // Do the actual migration + Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName); + Collection<String> migrated = transformPrivileges(privileges); + + if (!migrated.isEmpty()) { + LOGGER.info("{} For role {} migrating privileges from {} to {}", getDryRunMessage(), + roleName, privileges, migrated); + if (!dryRun) { + rolesSection.put(roleName, PrivilegeUtils.fromPrivilegeStrings(migrated)); + } + } + + roles.add(roleName); + } + } + } + + if (!dryRun) { + Path targetFile = new Path (outputFile.get()); + PolicyFiles.writeToPath(output, targetFile.getFileSystem(conf), targetFile); + LOGGER.info("Successfully saved migrated Sentry policy file at {}", outputFile.get()); + } + } + + private String getDryRunMessage() { + return dryRun ? "[Dry Run]" : ""; + } + + private Configuration getSentryConf() { + Configuration conf = new Configuration(); + if (confPath.isPresent()) { + conf.addResource(new Path(confPath.get()), true); + } + return conf; + } + + @VisibleForTesting + public boolean executeConfigTool(String [] args) throws Exception { + boolean result = true; + if (parseArgs(args)) { + run(); + } else { + result = false; + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java new file mode 100644 index 0000000..c24ae93 --- /dev/null +++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/PermissionsMigrationToolSolr.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.cli.tools; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.sentry.core.common.utils.SentryConstants; +import org.apache.sentry.core.model.solr.validator.SolrPrivilegeValidator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class provides SOLR specific functionality required for migrating Sentry privileges. + */ +public class PermissionsMigrationToolSolr extends PermissionsMigrationToolCommon { + private static final Logger LOGGER = LoggerFactory.getLogger(PermissionsMigrationToolSolr.class); + + + @Override + protected String getComponent(Configuration conf) { + return "SOLR"; + } + + @Override + protected String getServiceName(Configuration conf) { + return conf.get(SOLR_SERVICE_NAME, "service1"); + } + + @Override + protected Collection<String> transformPrivileges(Collection<String> privileges) { + List<String> result = new ArrayList<>(); + boolean migrated = false; + + if (getSourceVersion().major == 1) { // Migrate only Sentry 1.x permissions + for (String p : privileges) { + SolrPrivilegeValidator v = new SolrPrivilegeValidator(); + v.validate(p, false); + + if ("collection".equalsIgnoreCase(v.getEntityType()) && "admin".equalsIgnoreCase(v.getEntityName())) { + result.add(getPermissionStr("admin", "collections", v.getActionName())); + result.add(getPermissionStr("admin", "cores", v.getActionName())); + migrated = true; + } else if ("collection".equalsIgnoreCase(v.getEntityType()) && "*".equals(v.getEntityName())) { + result.add(getPermissionStr("admin", "collections", v.getActionName())); + result.add(getPermissionStr("admin", "cores", v.getActionName())); + result.add(p); + migrated = true; + } else { + result.add(p); + } + } + } + + return migrated ? result : Collections.emptyList(); + } + + private String getPermissionStr (String entityType, String entityName, String action) { + StringBuilder builder = new StringBuilder(); + builder.append(entityType); + builder.append(SentryConstants.KV_SEPARATOR); + builder.append(entityName); + if (action != null) { + builder.append(SentryConstants.AUTHORIZABLE_SEPARATOR); + builder.append(SentryConstants.PRIVILEGE_NAME); + builder.append(SentryConstants.KV_SEPARATOR); + builder.append(action); + } + return builder.toString(); + } + + public static void main(String[] args) throws Exception { + PermissionsMigrationToolSolr solrTool = new PermissionsMigrationToolSolr(); + try { + solrTool.executeConfigTool(args); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + Throwable current = e; + // find the first printable message; + while (current != null && current.getMessage() == null) { + current = current.getCause(); + } + String error = ""; + if (current != null && current.getMessage() != null) { + error = "Message: " + current.getMessage(); + } + System.out.println("The operation failed. " + error); + System.exit(1); + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java new file mode 100644 index 0000000..2d4f973 --- /dev/null +++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolCommon.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.cli.tools; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.Parser; + +abstract public class SentryConfigToolCommon { + private String policyFile; + private boolean validate; + private boolean importPolicy; + private boolean checkCompat; + private String confPath; + + /** + * parse arguments + * <pre> + * -conf,--sentry_conf <filepath> sentry config file path + * -p,--policy_ini <arg> policy file path + * -v,--validate validate policy file + * -c,--checkcompat check compatibility with service + * -i,--import import policy file + * -h,--help print usage + * </pre> + * @param args + */ + protected boolean parseArgs(String [] args) { + Options options = new Options(); + + Option globalPolicyPath = new Option("p", "policy_ini", true, + "Policy file path"); + globalPolicyPath.setRequired(true); + options.addOption(globalPolicyPath); + + Option validateOpt = new Option("v", "validate", false, + "Validate policy file"); + validateOpt.setRequired(false); + options.addOption(validateOpt); + + Option checkCompatOpt = new Option("c","checkcompat",false, + "Check compatibility with Sentry Service"); + checkCompatOpt.setRequired(false); + options.addOption(checkCompatOpt); + + Option importOpt = new Option("i", "import", false, + "Import policy file"); + importOpt.setRequired(false); + options.addOption(importOpt); + + // file path of sentry-site + Option sentrySitePathOpt = new Option("conf", "sentry_conf", true, "sentry-site file path"); + sentrySitePathOpt.setRequired(true); + options.addOption(sentrySitePathOpt); + + // help option + Option helpOpt = new Option("h", "help", false, "Shell usage"); + helpOpt.setRequired(false); + options.addOption(helpOpt); + + // this Options is parsed first for help option + Options helpOptions = new Options(); + helpOptions.addOption(helpOpt); + + try { + Parser parser = new GnuParser(); + + // parse help option first + CommandLine cmd = parser.parse(helpOptions, args, true); + for (Option opt : cmd.getOptions()) { + if (opt.getOpt().equals("h")) { + // get the help option, print the usage and exit + usage(options); + return false; + } + } + + // without help option + cmd = parser.parse(options, args); + + for (Option opt : cmd.getOptions()) { + if (opt.getOpt().equals("p")) { + policyFile = opt.getValue(); + } else if (opt.getOpt().equals("v")) { + validate = true; + } else if (opt.getOpt().equals("i")) { + importPolicy = true; + } else if (opt.getOpt().equals("c")) { + checkCompat = true; + } else if (opt.getOpt().equals("conf")) { + confPath = opt.getValue(); + } + } + + if (!validate && !importPolicy) { + throw new IllegalArgumentException("No action specified; at least one of action or import must be specified"); + } + } catch (ParseException pe) { + System.out.println(pe.getMessage()); + usage(options); + return false; + } + return true; + } + + // print usage + private void usage(Options sentryOptions) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("sentryConfigTool", sentryOptions); + } + + public abstract void run() throws Exception; + + @VisibleForTesting + public boolean executeConfigTool(String [] args) throws Exception { + boolean result = true; + if (parseArgs(args)) { + run(); + } else { + result = false; + } + return result; + } + + public String getPolicyFile() { return policyFile; } + public boolean getValidate() { return validate; } + public boolean getImportPolicy() { return importPolicy; } + public boolean getCheckCompat() { return checkCompat; } + public String getConfPath() { return confPath; } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java new file mode 100644 index 0000000..712d9ed --- /dev/null +++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolIndexer.java @@ -0,0 +1,341 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.cli.tools; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.collect.Table; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.sentry.core.common.Action; +import org.apache.sentry.core.common.exception.SentryConfigurationException; +import org.apache.sentry.core.common.utils.KeyValue; +import org.apache.sentry.core.model.indexer.IndexerPrivilegeModel; +import org.apache.sentry.provider.common.ProviderBackend; +import org.apache.sentry.provider.common.ProviderBackendContext; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory; +import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter; +import org.apache.sentry.provider.file.SimpleFileProviderBackend; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.apache.sentry.core.common.utils.SentryConstants.AUTHORIZABLE_SPLITTER; +import static org.apache.sentry.provider.common.AuthorizationComponent.HBASE_INDEXER; +import static org.apache.sentry.service.thrift.ServiceConstants.ClientConfig.SERVICE_NAME; + +/** + * SentryConfigToolIndexer is an administrative tool used to parse a HBase Indexer policy file + * and add the role, group mappings, and privileges therein to the Sentry service. + */ +public class SentryConfigToolIndexer { + + private static final Logger LOGGER = LoggerFactory.getLogger(SentryConfigToolIndexer.class); + + private String policyFile; + private boolean validate; + private boolean importPolicy; + private boolean checkCompat; + private String confPath; + + private String serviceName; + + + + public String getPolicyFile() { return policyFile; } + + public boolean getValidate() { return validate; } + public boolean getImportPolicy() { return importPolicy; } + public boolean getCheckCompat() { return checkCompat; } + public String getConfPath() { return confPath; } + public String getServiceName() { + return serviceName; + } + + /** + * Adds command line options for the tool to the passed Options object. Used to extend existing options. + * @param options + */ + public void setupOptions(Options options) { + Option globalPolicyPath = new Option("f", "policy_ini", true, + "Policy file path"); + globalPolicyPath.setRequired(false); + options.addOption(globalPolicyPath); + + Option validateOpt = new Option("v", "validate", false, + "Validate policy file"); + validateOpt.setRequired(false); + options.addOption(validateOpt); + + Option checkCompatOpt = new Option("c","checkcompat",false, + "Check compatibility with Sentry Service"); + checkCompatOpt.setRequired(false); + options.addOption(checkCompatOpt); + + Option importOpt = new Option("i", "import", false, + "Import policy file"); + importOpt.setRequired(false); + options.addOption(importOpt); + + } + + /** + * Parses and processes the arguments from the given command line object. + * @param cmd + */ + public void parseOptions(CommandLine cmd) { + boolean isToolActive = false; + for (Option opt : cmd.getOptions()) { + if (opt.getOpt().equals("mgr")) { + isToolActive = true; + } + } + if (!isToolActive) { + return; + } + for (Option opt : cmd.getOptions()) { + if (opt.getOpt().equals("f")) { + policyFile = opt.getValue(); + } else if (opt.getOpt().equals("v")) { + validate = true; + } else if (opt.getOpt().equals("i")) { + importPolicy = true; + } else if (opt.getOpt().equals("c")) { + checkCompat = true; + } else if (opt.getOpt().equals("conf")) { + confPath = opt.getValue(); + } else if (opt.getOpt().equals("s")) { + serviceName = opt.getValue(); + } + } + if (policyFile == null) { + throw new IllegalArgumentException("Missing required option: f"); + } + if (!validate && !importPolicy) { + throw new IllegalArgumentException("No action specified; at least one of action or import must be specified"); + } + } + + + /** + * Processes the necessary command based on the arguments parsed earlier. + * @throws Exception + */ + public void run() throws Exception { + String component = HBASE_INDEXER; + Configuration conf = getSentryConf(); + + String service = conf.get(SERVICE_NAME, getServiceName()); + + if (service == null) { + throw new IllegalArgumentException("Service was not defined. Please, use -s command option, or sentry.provider.backend.generic.service-name configuration entry."); + } + + LOGGER.info(String.format("Context: component=%s, service=%s", component, service)); + // instantiate a solr client for sentry service. This sets the ugi, so must + // be done before getting the ugi below. + try(SentryGenericServiceClient client = + SentryGenericServiceClientFactory.create(conf)) { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + String requestorName = ugi.getShortUserName(); + + convertINIToSentryServiceCmds(component, service, requestorName, conf, client, + getPolicyFile(), getValidate(), getImportPolicy(), getCheckCompat()); + } + } + + private Configuration getSentryConf() { + Configuration conf = new Configuration(); + conf.addResource(new Path(getConfPath()), true); + return conf; + } + + private void convertINIToSentryServiceCmds(String component, + String service, String requestorName, + Configuration conf, SentryGenericServiceClient client, + String policyFile, boolean validate, boolean importPolicy, + boolean checkCompat) throws Exception { + + //instantiate a file providerBackend for parsing + LOGGER.info("Reading policy file at: " + policyFile); + SimpleFileProviderBackend policyFileBackend = + new SimpleFileProviderBackend(conf, policyFile); + ProviderBackendContext context = new ProviderBackendContext(); + context.setValidators(IndexerPrivilegeModel.getInstance().getPrivilegeValidators()); + policyFileBackend.initialize(context); + if (validate) { + validatePolicy(policyFileBackend); + } + + if (checkCompat) { + checkCompat(policyFileBackend); + } + + //import the relations about group,role and privilege into the DB store + Set<String> roles = Sets.newHashSet(); + Table<String, String, Set<String>> groupRolePrivilegeTable = + policyFileBackend.getGroupRolePrivilegeTable(); + GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, service, false); + + for (String groupName : groupRolePrivilegeTable.rowKeySet()) { + for (String roleName : groupRolePrivilegeTable.columnKeySet()) { + if (!roles.contains(roleName)) { + LOGGER.info(dryRunMessage(importPolicy) + "Creating role: " + roleName.toLowerCase(Locale.US)); + if (importPolicy) { + client.createRoleIfNotExist(requestorName, roleName, component); + } + roles.add(roleName); + } + + Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName); + if (privileges == null) { + continue; + } + LOGGER.info(dryRunMessage(importPolicy) + "Adding role: " + roleName.toLowerCase(Locale.US) + " to group: " + groupName); + if (importPolicy) { + client.grantRoleToGroups(requestorName, roleName, component, Sets.newHashSet(groupName)); + } + + for (String permission : privileges) { + String action = null; + + for (String authorizable : AUTHORIZABLE_SPLITTER. + trimResults().split(permission)) { + KeyValue kv = new KeyValue(authorizable); + String key = kv.getKey(); + String value = kv.getValue(); + if ("action".equalsIgnoreCase(key)) { + action = value; + } + } + + // Service doesn't support not specifying action + if (action == null) { + permission += "->action=" + Action.ALL; + } + LOGGER.info(dryRunMessage(importPolicy) + "Adding permission: " + permission + " to role: " + roleName.toLowerCase(Locale.US)); + if (importPolicy) { + client.grantPrivilege(requestorName, roleName, component, converter.fromString(permission)); + } + } + } + } + } + + private void validatePolicy(ProviderBackend backend) throws Exception { + try { + backend.validatePolicy(true); + } catch (SentryConfigurationException e) { + printConfigErrorsWarnings(e); + throw e; + } + } + + private void printConfigErrorsWarnings(SentryConfigurationException configException) { + System.out.println(" *** Found configuration problems *** "); + for (String errMsg : configException.getConfigErrors()) { + System.out.println("ERROR: " + errMsg); + } + for (String warnMsg : configException.getConfigWarnings()) { + System.out.println("Warning: " + warnMsg); + } + } + + private void checkCompat(SimpleFileProviderBackend backend) throws Exception { + Map<String, Set<String>> rolesCaseMapping = new HashMap<String, Set<String>>(); + Table<String, String, Set<String>> groupRolePrivilegeTable = + backend.getGroupRolePrivilegeTable(); + + for (String roleName : groupRolePrivilegeTable.columnKeySet()) { + String roleNameLower = roleName.toLowerCase(Locale.US); + if (!roleName.equals(roleNameLower)) { + if (!rolesCaseMapping.containsKey(roleNameLower)) { + rolesCaseMapping.put(roleNameLower, Sets.newHashSet(roleName)); + } else { + rolesCaseMapping.get(roleNameLower).add(roleName); + } + } + } + + List<String> errors = new LinkedList<String>(); + StringBuilder warningString = new StringBuilder(); + if (!rolesCaseMapping.isEmpty()) { + warningString.append("The following roles names will be lower cased when added to the Sentry Service.\n"); + warningString.append("This will cause document-level security to fail to match the role tokens.\n"); + warningString.append("Role names: "); + } + boolean firstWarning = true; + + for (Map.Entry<String, Set<String>> entry : rolesCaseMapping.entrySet()) { + Set<String> caseMapping = entry.getValue(); + if (caseMapping.size() > 1) { + StringBuilder errorString = new StringBuilder(); + errorString.append("The following (cased) roles map to the same role in the sentry service: "); + boolean first = true; + for (String casedRole : caseMapping) { + errorString.append(first ? "" : ", "); + errorString.append(casedRole); + first = false; + } + errorString.append(". Role in service: ").append(entry.getKey()); + errors.add(errorString.toString()); + } + + for (String casedRole : caseMapping) { + warningString.append(firstWarning? "" : ", "); + warningString.append(casedRole); + firstWarning = false; + } + } + + for (String error : errors) { + System.out.println("ERROR: " + error); + } + System.out.println("\n"); + + System.out.println("Warning: " + warningString.toString()); + if (errors.size() > 0) { + SentryConfigurationException ex = + new SentryConfigurationException("Compatibility check failure"); + ex.setConfigErrors(errors); + ex.setConfigWarnings(Lists.<String>asList(warningString.toString(), new String[0])); + throw ex; + } + } + + private String dryRunMessage(boolean importPolicy) { + if (importPolicy) { + return ""; + } else { + return "[Dry Run] "; + } + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java new file mode 100644 index 0000000..dfa981d --- /dev/null +++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentryConfigToolSolr.java @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.cli.tools; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.collect.Table; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.sentry.core.common.Action; +import org.apache.sentry.core.common.exception.SentryConfigurationException; +import org.apache.sentry.core.common.utils.KeyValue; +import org.apache.sentry.core.common.utils.SentryConstants; +import org.apache.sentry.core.model.solr.SolrPrivilegeModel; +import org.apache.sentry.provider.common.ProviderBackend; +import org.apache.sentry.provider.common.ProviderBackendContext; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory; +import org.apache.sentry.provider.db.generic.tools.GenericPrivilegeConverter; +import org.apache.sentry.provider.file.SimpleFileProviderBackend; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * SentryConfigToolSolr is an administrative tool used to parse a Solr policy file + * and add the role, group mappings, and privileges therein to the Sentry service. + */ +public class SentryConfigToolSolr extends SentryConfigToolCommon { + + private static final Logger LOGGER = LoggerFactory.getLogger(SentryConfigToolSolr.class); + public static final String SOLR_SERVICE_NAME = "sentry.service.client.solr.service.name"; + + @Override + public void run() throws Exception { + String component = "SOLR"; + Configuration conf = getSentryConf(); + + String service = conf.get(SOLR_SERVICE_NAME, "service1"); + // instantiate a solr client for sentry service. This sets the ugi, so must + // be done before getting the ugi below. + try(SentryGenericServiceClient client = + SentryGenericServiceClientFactory.create(conf)) { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + String requestorName = ugi.getShortUserName(); + + convertINIToSentryServiceCmds(component, service, requestorName, conf, client, + getPolicyFile(), getValidate(), getImportPolicy(), getCheckCompat()); + } + } + + private Configuration getSentryConf() { + Configuration conf = new Configuration(); + conf.addResource(new Path(getConfPath()), true); + return conf; + } + + /** + * Convert policy file to solrctl commands -- based on SENTRY-480 + */ + private void convertINIToSentryServiceCmds(String component, + String service, String requestorName, + Configuration conf, SentryGenericServiceClient client, + String policyFile, boolean validate, boolean importPolicy, + boolean checkCompat) throws Exception { + + //instantiate a file providerBackend for parsing + LOGGER.info("Reading policy file at: " + policyFile); + SimpleFileProviderBackend policyFileBackend = + new SimpleFileProviderBackend(conf, policyFile); + ProviderBackendContext context = new ProviderBackendContext(); + context.setValidators(SolrPrivilegeModel.getInstance().getPrivilegeValidators()); + policyFileBackend.initialize(context); + if (validate) { + validatePolicy(policyFileBackend); + } + + if (checkCompat) { + checkCompat(policyFileBackend); + } + + //import the relations about group,role and privilege into the DB store + Set<String> roles = Sets.newHashSet(); + Table<String, String, Set<String>> groupRolePrivilegeTable = + policyFileBackend.getGroupRolePrivilegeTable(); + GenericPrivilegeConverter converter = new GenericPrivilegeConverter(component, service, false); + + for (String groupName : groupRolePrivilegeTable.rowKeySet()) { + for (String roleName : groupRolePrivilegeTable.columnKeySet()) { + if (!roles.contains(roleName)) { + LOGGER.info(dryRunMessage(importPolicy) + "Creating role: " + roleName.toLowerCase(Locale.US)); + if (importPolicy) { + client.createRoleIfNotExist(requestorName, roleName, component); + } + roles.add(roleName); + } + + Set<String> privileges = groupRolePrivilegeTable.get(groupName, roleName); + if (privileges == null) { + continue; + } + LOGGER.info(dryRunMessage(importPolicy) + "Adding role: " + roleName.toLowerCase(Locale.US) + " to group: " + groupName); + if (importPolicy) { + client.grantRoleToGroups(requestorName, roleName, component, Sets.newHashSet(groupName)); + } + + for (String permission : privileges) { + String action = null; + + for (String authorizable : SentryConstants.AUTHORIZABLE_SPLITTER. + trimResults().split(permission)) { + KeyValue kv = new KeyValue(authorizable); + String key = kv.getKey(); + String value = kv.getValue(); + if ("action".equalsIgnoreCase(key)) { + action = value; + } + } + + // Service doesn't support not specifying action + if (action == null) { + permission += "->action=" + Action.ALL; + } + LOGGER.info(dryRunMessage(importPolicy) + "Adding permission: " + permission + " to role: " + roleName.toLowerCase(Locale.US)); + if (importPolicy) { + client.grantPrivilege(requestorName, roleName, component, converter.fromString(permission)); + } + } + } + } + } + + private void validatePolicy(ProviderBackend backend) throws Exception { + try { + backend.validatePolicy(true); + } catch (SentryConfigurationException e) { + printConfigErrorsWarnings(e); + throw e; + } + } + + private void printConfigErrorsWarnings(SentryConfigurationException configException) { + System.out.println(" *** Found configuration problems *** "); + for (String errMsg : configException.getConfigErrors()) { + System.out.println("ERROR: " + errMsg); + } + for (String warnMsg : configException.getConfigWarnings()) { + System.out.println("Warning: " + warnMsg); + } + } + + private void checkCompat(SimpleFileProviderBackend backend) throws Exception { + Map<String, Set<String>> rolesCaseMapping = new HashMap<String, Set<String>>(); + Table<String, String, Set<String>> groupRolePrivilegeTable = + backend.getGroupRolePrivilegeTable(); + + for (String roleName : groupRolePrivilegeTable.columnKeySet()) { + String roleNameLower = roleName.toLowerCase(Locale.US); + if (!roleName.equals(roleNameLower)) { + if (!rolesCaseMapping.containsKey(roleNameLower)) { + rolesCaseMapping.put(roleNameLower, Sets.newHashSet(roleName)); + } else { + rolesCaseMapping.get(roleNameLower).add(roleName); + } + } + } + + List<String> errors = new LinkedList<String>(); + StringBuilder warningString = new StringBuilder(); + if (!rolesCaseMapping.isEmpty()) { + warningString.append("The following roles names will be lower cased when added to the Sentry Service.\n"); + warningString.append("This will cause document-level security to fail to match the role tokens.\n"); + warningString.append("Role names: "); + } + boolean firstWarning = true; + + for (Map.Entry<String, Set<String>> entry : rolesCaseMapping.entrySet()) { + Set<String> caseMapping = entry.getValue(); + if (caseMapping.size() > 1) { + StringBuilder errorString = new StringBuilder(); + errorString.append("The following (cased) roles map to the same role in the sentry service: "); + boolean first = true; + for (String casedRole : caseMapping) { + errorString.append(first ? "" : ", "); + errorString.append(casedRole); + first = false; + } + errorString.append(". Role in service: ").append(entry.getKey()); + errors.add(errorString.toString()); + } + + for (String casedRole : caseMapping) { + warningString.append(firstWarning? "" : ", "); + warningString.append(casedRole); + firstWarning = false; + } + } + + for (String error : errors) { + System.out.println("ERROR: " + error); + } + System.out.println("\n"); + + System.out.println("Warning: " + warningString.toString()); + if (errors.size() > 0) { + SentryConfigurationException ex = + new SentryConfigurationException("Compatibility check failure"); + ex.setConfigErrors(errors); + ex.setConfigWarnings(Lists.<String>asList(warningString.toString(), new String[0])); + throw ex; + } + } + + private String dryRunMessage(boolean importPolicy) { + if (importPolicy) { + return ""; + } else { + return "[Dry Run] "; + } + } + + public static void main(String[] args) throws Exception { + SentryConfigToolSolr solrTool = new SentryConfigToolSolr(); + try { + solrTool.executeConfigTool(args); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + Throwable current = e; + // find the first printable message; + while (current != null && current.getMessage() == null) { + current = current.getCause(); + } + String error = ""; + if (current != null && current.getMessage() != null) { + error = "Message: " + current.getMessage(); + } + System.out.println("The operation failed. " + error); + System.exit(1); + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/6752f14a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java ---------------------------------------------------------------------- diff --git a/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java new file mode 100644 index 0000000..3723fd9 --- /dev/null +++ b/sentry-tools/src/main/java/org/apache/sentry/cli/tools/SentrySchemaHelper.java @@ -0,0 +1,315 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.cli.tools; + +import java.util.IllegalFormatException; + +public final class SentrySchemaHelper { + public static final String DB_DERBY = "derby"; + public static final String DB_MYSQL = "mysql"; + public static final String DB_POSTGRACE = "postgres"; + public static final String DB_ORACLE = "oracle"; + public static final String DB_DB2 = "db2"; + + public interface NestedScriptParser { + + public enum CommandType { + PARTIAL_STATEMENT, + TERMINATED_STATEMENT, + COMMENT + } + + String DEFAUTL_DELIMITER = ";"; + /*** + * Find the type of given command + * @param dbCommand + * @return + */ + boolean isPartialCommand(String dbCommand) throws IllegalArgumentException; + + /** Parse the DB specific nesting format and extract the inner script name if any + * @param dbCommand command from parent script + * @return + * @throws IllegalFormatException + */ + String getScriptName(String dbCommand) throws IllegalArgumentException; + + /*** + * Find if the given command is a nested script execution + * @param dbCommand + * @return + */ + boolean isNestedScript(String dbCommand); + + /*** + * Find if the given command is should be passed to DB + * @param dbCommand + * @return + */ + boolean isNonExecCommand(String dbCommand); + + /*** + * Get the SQL statement delimiter + * @return + */ + String getDelimiter(); + + /*** + * Clear any client specific tags + * @return + */ + String cleanseCommand(String dbCommand); + + /*** + * Does the DB required table/column names quoted + * @return + */ + boolean needsQuotedIdentifier(); + + /*** + * Set DB specific options if any + * @param dbOps + */ + void setDbOpts(String dbOps); + } + + + /*** + * Base implemenation of NestedScriptParser + * abstractCommandParser. + * + */ + private static abstract class AbstractCommandParser implements NestedScriptParser { + private String dbOpts = null; + + @Override + public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ + if (dbCommand == null || dbCommand.isEmpty()) { + throw new IllegalArgumentException("invalid command line " + dbCommand); + } + String trimmedDbCommand = dbCommand.trim(); + return !(trimmedDbCommand.endsWith(getDelimiter()) || isNonExecCommand(trimmedDbCommand)); + } + + @Override + public boolean isNonExecCommand(String dbCommand) { + return dbCommand.startsWith("--") || dbCommand.startsWith("#"); + } + + @Override + public String getDelimiter() { + return DEFAUTL_DELIMITER; + } + + @Override + public String cleanseCommand(String dbCommand) { + // strip off the delimiter + if (dbCommand.endsWith(getDelimiter())) { + dbCommand = dbCommand.substring(0, + dbCommand.length() - getDelimiter().length()); + } + return dbCommand; + } + + @Override + public boolean needsQuotedIdentifier() { + return false; + } + + @Override + public void setDbOpts(String dbOpts) { + this.dbOpts = dbOpts; + } + + protected String getDbOpts() { + return dbOpts; + } + } + + + // Derby commandline parser + public static class DerbyCommandParser extends AbstractCommandParser { + private static final String DERBY_NESTING_TOKEN = "RUN"; + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + + if (!isNestedScript(dbCommand)) { + throw new IllegalArgumentException("Not a script format " + dbCommand); + } + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + return tokens[1].replace(";", "").replaceAll("'", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + // Derby script format is RUN '<file>' + return dbCommand.startsWith(DERBY_NESTING_TOKEN); + } + } + + + // MySQL parser + public static class MySqlCommandParser extends AbstractCommandParser { + private static final String MYSQL_NESTING_TOKEN = "SOURCE"; + private static final String DELIMITER_TOKEN = "DELIMITER"; + private String delimiter = DEFAUTL_DELIMITER; + + @Override + public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{ + boolean isPartial = super.isPartialCommand(dbCommand); + // if this is a delimiter directive, reset our delimiter + if (dbCommand.startsWith(DELIMITER_TOKEN)) { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + delimiter = tokens[1]; + } + return isPartial; + } + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + // remove ending ';' + return tokens[1].replace(";", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(MYSQL_NESTING_TOKEN); + } + + @Override + public String getDelimiter() { + return delimiter; + } + + @Override + public boolean isNonExecCommand(String dbCommand) { + return super.isNonExecCommand(dbCommand) || + dbCommand.startsWith("/*") && dbCommand.endsWith("*/") || + dbCommand.startsWith(DELIMITER_TOKEN); + } + + @Override + public String cleanseCommand(String dbCommand) { + return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", ""); + } + + } + + // Postgres specific parser + public static class PostgresCommandParser extends AbstractCommandParser { + public static final String POSTGRES_STRING_COMMAND_FILTER = "SET standard_conforming_strings"; + public static final String POSTGRES_STRING_CLIENT_ENCODING = "SET client_encoding"; + public static final String POSTGRES_SKIP_STANDARD_STRING = "postgres.filter.81"; + private static final String POSTGRES_NESTING_TOKEN = "\\i"; + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + String[] tokens = dbCommand.split(" "); + if (tokens.length != 2) { + throw new IllegalArgumentException("Couldn't parse line " + dbCommand); + } + // remove ending ';' + return tokens[1].replace(";", ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(POSTGRES_NESTING_TOKEN); + } + + @Override + public boolean needsQuotedIdentifier() { + return true; + } + + @Override + public boolean isNonExecCommand(String dbCommand) { + // Skip "standard_conforming_strings" command which is not supported in older postgres + if (POSTGRES_SKIP_STANDARD_STRING.equalsIgnoreCase(getDbOpts()) + && (dbCommand.startsWith(POSTGRES_STRING_COMMAND_FILTER) || dbCommand.startsWith(POSTGRES_STRING_CLIENT_ENCODING))) { + return true; + } + return super.isNonExecCommand(dbCommand); + } + } + + //Oracle specific parser + public static class OracleCommandParser extends AbstractCommandParser { + private static final String ORACLE_NESTING_TOKEN = "@"; + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + if (!isNestedScript(dbCommand)) { + throw new IllegalArgumentException("Not a nested script format " + dbCommand); + } + // remove ending ';' and starting '@' + return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, ""); + } + + @Override + public boolean isNestedScript(String dbCommand) { + return dbCommand.startsWith(ORACLE_NESTING_TOKEN); + } + } + + // DB2 commandline parser + public static class DB2CommandParser extends AbstractCommandParser { + + @Override + public String getScriptName(String dbCommand) throws IllegalArgumentException { + //DB2 does not support nesting script + throw new IllegalArgumentException("DB2 does not support nesting script " + dbCommand); + } + + @Override + public boolean isNestedScript(String dbCommand) { + //DB2 does not support nesting script + return false; + } + } + + public static NestedScriptParser getDbCommandParser(String dbName) { + if (dbName.equalsIgnoreCase(DB_DERBY)) { + return new DerbyCommandParser(); + } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { + return new MySqlCommandParser(); + } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { + return new PostgresCommandParser(); + } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { + return new OracleCommandParser(); + } else if (dbName.equalsIgnoreCase(DB_DB2)) { + return new DB2CommandParser(); + } else { + throw new IllegalArgumentException("Unknown dbType " + dbName); + } + } + + private SentrySchemaHelper() { + // Make constructor private to avoid instantiation + } +}
