Repository: ambari Updated Branches: refs/heads/branch-2.5 b1438eea9 -> fb744009e
AMBARI-21577. Hive-Service check failing in post EU validation (BI-HDP). (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fb744009 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fb744009 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fb744009 Branch: refs/heads/branch-2.5 Commit: fb744009e8d6e269e78943750ec09c73f133470d Parents: b1438ee Author: Siddharth Wagle <[email protected]> Authored: Fri Jul 28 08:44:58 2017 -0700 Committer: Siddharth Wagle <[email protected]> Committed: Fri Jul 28 08:44:58 2017 -0700 ---------------------------------------------------------------------- .../upgrades/FixAuthToLocalMappingAction.java | 149 +++++++++++++++++++ .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 8 + .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 8 + .../FixAuthToLocalMappingActionTest.java | 126 ++++++++++++++++ 4 files changed, 291 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java new file mode 100644 index 0000000..e62f2db --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.kerberos.KerberosDescriptor; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; + +import com.google.inject.Inject; + +/** + * Fixes auth_to_local rules during upgrade from IOP to HDP. An example of + * invalid rule introduced by HBASE_REST_SERVER in the IOP stack, set auth to + * local mapping for HTTP spnego principal to local hbase user, which needs to + * be deleted for HIVE service check to pass. + */ +public class FixAuthToLocalMappingAction extends AbstractServerAction { + + private static final String SPNEGO_PRINC_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(HTTP@.*\\)s/\\.\\*/.*/\\n"; + private static final String AMS_HBASE_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(amshbase@.*\\)s/\\.\\*/%s/\\n"; + private static final String ZK_AMS_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(zookeeper@.*\\)s/\\.\\*/%s/\\n"; + + @Inject + private Clusters clusters; + + @Inject + private KerberosHelper kerberosHelper; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) + throws AmbariException, InterruptedException { + + String clusterName = getExecutionCommand().getClusterName(); + Cluster cluster = clusters.getCluster(clusterName); + + KerberosDescriptor kd = kerberosHelper.getKerberosDescriptor(cluster); + if (kd == null) { + return null; + } + + Map<String, Set<String>> configProperties = new HashMap<>(); + for (String property : kd.getAllAuthToLocalProperties()) { + if (!StringUtils.isEmpty(property) && property.contains("/")) { + String[] propertyParts = property.split("/"); + if (configProperties.containsKey(propertyParts[0])) { + configProperties.get(propertyParts[0]).add(propertyParts[1]); + } else { + Set<String> properties = new HashSet<>(); + properties.add(propertyParts[1]); + configProperties.put(propertyParts[0], properties); + } + } + } + + String hbaseUser = null; + Config hbaseEnv = cluster.getDesiredConfigByType("hbase-env"); + if (hbaseEnv != null) { + Map<String, String> properties = hbaseEnv.getProperties(); + if (!MapUtils.isEmpty(properties)) { + hbaseUser = properties.get("hbase_user"); + } + } + + String amsUser = null; + Config amsEnv = cluster.getDesiredConfigByType("ams-env"); + if (amsEnv != null) { + Map<String, String> properties = amsEnv.getProperties(); + if (!MapUtils.isEmpty(properties)) { + amsUser = properties.get("ambari_metrics_user"); + } + } + + boolean replaced = false; + StringBuilder message = new StringBuilder("Replaced offending auto_to_local mappings"); + + for (Map.Entry<String, Set<String>> configProperty : configProperties.entrySet()) { + String configType = configProperty.getKey(); + Config config = cluster.getDesiredConfigByType(configType); + + if (config == null) { + continue; + } + + for (String property : configProperty.getValue()) { + Map<String, String> properties = config.getProperties(); + if (!MapUtils.isEmpty(properties) && properties.containsKey(property)) { + String authToLocalRules = properties.get(property); + if (!StringUtils.isEmpty(authToLocalRules)) { + authToLocalRules = authToLocalRules.replaceAll(SPNEGO_PRINC_PATTERN, ""); + if (hbaseUser != null) { + authToLocalRules = authToLocalRules.replaceAll(String.format(AMS_HBASE_PATTERN, hbaseUser), ""); + } + if (amsUser != null) { + authToLocalRules = authToLocalRules.replaceAll(String.format(ZK_AMS_PATTERN, amsUser), ""); + } + // Only if something was replaced + if (!properties.get(property).equals(authToLocalRules)) { + properties.put(property, authToLocalRules); + message.append(" , property => "); + message.append(property); + message.append(" , config => "); + message.append(configType); + + config.setProperties(properties); + config.save(); + replaced = true; + } + } + } + } + } + + String finalMessage = message.toString(); + if (!replaced) { + finalMessage = "No offending auto_to_local mappings found"; + } + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", finalMessage, ""); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index 9d3cc1e..a7fdeec 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -222,6 +222,14 @@ </task> </execute-stage> + <!-- core-site --> + <execute-stage title="Fix auth_to_local mapping"> + <condition xsi:type="security" type="kerberos"/> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixAuthToLocalMappingAction"> + <summary>Fix auth to local mapping rules</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations"> <task xsi:type="configure" id="biginsights_4_2_yarn_config_update" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index 3637927..5532248 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -217,6 +217,14 @@ </task> </execute-stage> + <!-- core-site --> + <execute-stage title="Fix auth_to_local mapping"> + <condition xsi:type="security" type="kerberos"/> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixAuthToLocalMappingAction"> + <summary>Fix auth to local mapping rules</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations"> <task xsi:type="configure" id="biginsights_4_2_yarn_config_update" /> http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java new file mode 100644 index 0000000..ab102bc --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.powermock.api.easymock.PowerMock.verifyAll; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.controller.KerberosHelper; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.kerberos.KerberosDescriptor; +import org.easymock.Capture; +import org.junit.Test; + +import com.google.common.collect.Maps; +import com.google.inject.Injector; + +import junit.framework.Assert; + +public class FixAuthToLocalMappingActionTest { + + String authToLocalRulesOriginal = "RULE:[1:$1@$0]([email protected])s/.*/ambari-qa/\nRULE:[1:$1@$0]([email protected])s/.*/hbase/\nRULE:[1:$1@$0]([email protected])s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0]([email protected])s/.*/hbase/\nRULE:[2:$1@$0]([email protected])s/.*/ams/\nRULE:[2:$1@$0]([email protected])s/.*/hdfs/\nRULE:[2:$1@$0]([email protected])s/.*/hbase/\nRULE:[2:$1@$0]([email protected])s/.*/hive/\nRULE:[2:$1@$0]([email protected])s/.*/mapred/\nRULE:[2:$1@$0]([email protected])s/.*/yarn/\nRULE:[2:$1@$0]([email protected])s/.*/hdfs/\nRULE:[2:$1@$0]([email protected])s/.*/yarn/\nRULE:[2:$1@$0]([email protected])s/.*/yarn/\nRULE:[2:$1@$0]([email protected])s/.*/ams/\nRULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\nRULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\nRULE:[2:$1@$0](hm@.*)s/.*/hbase/\nRULE:[2:$1@$0](jhs@.*)s/.*/mapred/\nRULE:[2:$1@$0](rs@.*)s/.*/hbase/\nDEFAULT"; + String authToLocalRulesUpdated = "RULE:[1:$1@$0]([email protected])s/.*/ambari-qa/\nRULE:[1:$1@$0]([email protected])s/.*/hbase/\nRULE:[1:$1@$0]([email protected])s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0]([email protected])s/.*/ams/\nRULE:[2:$1@$0]([email protected])s/.*/hdfs/\nRULE:[2:$1@$0]([email protected])s/.*/hbase/\nRULE:[2:$1@$0]([email protected])s/.*/hive/\nRULE:[2:$1@$0]([email protected])s/.*/mapred/\nRULE:[2:$1@$0]([email protected])s/.*/yarn/\nRULE:[2:$1@$0]([email protected])s/.*/hdfs/\nRULE:[2:$1@$0]([email protected])s/.*/yarn/\nRULE:[2:$1@$0]([email protected])s/.*/yarn/\nRULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\nRULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\nRULE:[2:$1@$0](hm@.*)s/.*/hbase/\nRULE:[2:$1@$0](jhs@.*)s/.*/mapred/\nRULE:[2:$1@$0](rs@.*)s/.*/hbase/\nDEFAULT"; + + @Test + public void testExecute() throws Exception { + String clusterName = "c1"; + + Injector injector = createNiceMock(Injector.class); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + KerberosHelper kerberosHelper = createNiceMock(KerberosHelper.class); + KerberosDescriptor descriptor = createNiceMock(KerberosDescriptor.class); + + expect(kerberosHelper.getKerberosDescriptor(cluster)).andReturn(descriptor).anyTimes(); + Set<String> mappings = new HashSet<>(); + mappings.add("core-site/hadoop.security.auth_to_local"); + + expect(descriptor.getAllAuthToLocalProperties()).andReturn(mappings); + + Map<String, String> commandParams = Maps.newHashMap(); + commandParams.put("clusterName", clusterName); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName(clusterName); + + HostRoleCommand hrc = createNiceMock(HostRoleCommand.class); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)); + + Config hbaseEnv = createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("hbase-env")).andReturn(hbaseEnv); + expect(hbaseEnv.getProperties()).andReturn(Collections.singletonMap("hbase_user", "hbase")); + + Config amsEnv = createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("ams-env")).andReturn(amsEnv); + expect(amsEnv.getProperties()).andReturn(Collections.singletonMap("ambari_metrics_user", "ams")); + + Config coreSite = createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("core-site")).andReturn(coreSite); + + Map<String, String> original = Maps.newHashMap(); + original.put("hadoop.security.auth_to_local", authToLocalRulesOriginal); + expect(coreSite.getProperties()).andReturn(original); + + Capture<Map<String, String>> updated = Capture.newInstance(); + coreSite.setProperties(capture(updated)); + expectLastCall(); + coreSite.save(); + expectLastCall(); + + expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes(); + expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce(); + + FixAuthToLocalMappingAction action = new FixAuthToLocalMappingAction(); + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + Field clustersField = FixAuthToLocalMappingAction.class.getDeclaredField("clusters"); + clustersField.setAccessible(true); + clustersField.set(action, clusters); + + Field kerberosHelperField = FixAuthToLocalMappingAction.class.getDeclaredField("kerberosHelper"); + kerberosHelperField.setAccessible(true); + kerberosHelperField.set(action, kerberosHelper); + + replay(kerberosHelper, descriptor, injector, clusters, cluster, hrc, hbaseEnv, amsEnv, coreSite); + + ConcurrentMap<String, Object> emptyMap = Maps.newConcurrentMap(); + action.execute(emptyMap); + + verifyAll(); + + Assert.assertEquals(authToLocalRulesUpdated, updated.getValue().get("hadoop.security.auth_to_local")); + } +}
