Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-21348 e36c11b68 -> fc93b2795
AMBARI-21408. Update Custom Action during EU to replace all occurrences of IOP/iop with HDP/hdp Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fc93b279 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fc93b279 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fc93b279 Branch: refs/heads/branch-feature-AMBARI-21348 Commit: fc93b27955fa1cccc775b229d730ad34884b7c9d Parents: e36c11b Author: Attila Doroszlai <[email protected]> Authored: Thu Jul 6 14:37:33 2017 +0200 Committer: Attila Doroszlai <[email protected]> Committed: Thu Jul 6 15:30:12 2017 +0200 ---------------------------------------------------------------------- .../upgrades/ChangeStackReferencesAction.java | 108 +++++++++++++++++++ .../ChangeStackRootDirectoryAction.java | 92 ---------------- .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 6 +- .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 6 +- .../ChangeStackReferencesActionTest.java | 102 ++++++++++++++++++ .../ChangeStackRootDirectoryActionTest.java | 101 ----------------- 6 files changed, 216 insertions(+), 199 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fc93b279/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesAction.java new file mode 100644 index 0000000..d75d031 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesAction.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; +import javax.inject.Inject; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.actionmanager.HostRoleStatus; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.serveraction.AbstractServerAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Function; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +/** + * Replaces references to the old stack ("iop" and "IOP") with the new + * stack ("hdp" and "HDP") during upgrade from BigInsights to HDP. + */ +public class ChangeStackReferencesAction extends AbstractServerAction { + + private static final Logger LOG = LoggerFactory.getLogger(ChangeStackReferencesAction.class); + private static final Set<Map.Entry<String, String>> REPLACEMENTS = Maps.asMap( + Sets.newHashSet("/usr/iop", "iop/apps", "iop.version", "IOP_VERSION"), + new Function<String, String>() { + @Nullable + @Override + public String apply(@Nullable String input) { + return input != null + ? input.replace("iop", "hdp").replace("IOP", "HDP") + : null; + } + } + ).entrySet(); + + @Inject + private Clusters clusters; + + @Override + public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException { + StringBuilder out = new StringBuilder(); + + String msg = "Changing stack-specific references from IOP to HDP"; + LOG.info(msg); + out.append(msg).append(System.lineSeparator()); + + String clusterName = getExecutionCommand().getClusterName(); + Cluster cluster = clusters.getCluster(clusterName); + + for (String configType: cluster.getDesiredConfigs().keySet()) { + Config config = cluster.getDesiredConfigByType(configType); + String typeMsg = String.format("Checking config type=%s version=%s tag=%s", config.getType(), config.getVersion(), config.getTag()); + LOG.debug(typeMsg); + out.append(typeMsg).append(System.lineSeparator()); + Map<String, String> properties = config.getProperties(); + if (!properties.isEmpty()) { + Map<String, String> changedProperties = Maps.newHashMap(); + for (Map.Entry<String, String> entry : properties.entrySet()) { + String key = entry.getKey(); + String original = entry.getValue(); + if (original != null) { + String replaced = original; + for (Map.Entry<String, String> replacement : REPLACEMENTS) { + replaced = replaced.replace(replacement.getKey(), replacement.getValue()); + } + if (!replaced.equals(original)) { + changedProperties.put(key, replaced); + String itemMsg = String.format("Changing %s", key); + LOG.debug(itemMsg); + out.append(itemMsg).append(System.lineSeparator()); + } + } + } + if (!changedProperties.isEmpty()) { + config.updateProperties(changedProperties); + config.save(); + } + } + } + + return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", out.toString(), ""); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fc93b279/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryAction.java deleted file mode 100644 index 31c4292..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryAction.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.serveraction.upgrades; - -import java.util.Map; -import java.util.concurrent.ConcurrentMap; - -import javax.inject.Inject; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.actionmanager.HostRoleStatus; -import org.apache.ambari.server.agent.CommandReport; -import org.apache.ambari.server.serveraction.AbstractServerAction; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Maps; - -/** - * Replaces references to the old stack root ("/usr/iop") with the new - * stack root ("/usr/hdp") during upgrade from BigInsights to HDP. - * TODO pass stack root locations as parameters from the upgrade pack - */ -public class ChangeStackRootDirectoryAction extends AbstractServerAction { - - private static final Logger LOG = LoggerFactory.getLogger(ChangeStackRootDirectoryAction.class); - private static final String OLD_STACK_ROOT = "/usr/iop"; - private static final String NEW_STACK_ROOT = "/usr/hdp"; - - @Inject - private Clusters clusters; - - @Override - public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException { - StringBuilder out = new StringBuilder(); - - String msg = String.format("Changing stack root directory references from %s to %s", OLD_STACK_ROOT, NEW_STACK_ROOT); - LOG.info(msg); - out.append(msg).append(System.lineSeparator()); - - String clusterName = getExecutionCommand().getClusterName(); - Cluster cluster = clusters.getCluster(clusterName); - - for (String configType: cluster.getDesiredConfigs().keySet()) { - Config config = cluster.getDesiredConfigByType(configType); - String typeMsg = String.format("Checking config type=%s version=%s tag=%s", config.getType(), config.getVersion(), config.getTag()); - LOG.debug(typeMsg); - out.append(typeMsg).append(System.lineSeparator()); - Map<String, String> properties = config.getProperties(); - if (!properties.isEmpty()) { - Map<String, String> changedProperties = Maps.newHashMap(); - for (Map.Entry<String, String> entry : properties.entrySet()) { - String key = entry.getKey(); - String original = entry.getValue(); - if (original != null) { - String replaced = original.replace(OLD_STACK_ROOT, NEW_STACK_ROOT); - if (!replaced.equals(original)) { - changedProperties.put(key, replaced); - String itemMsg = String.format("Changing %s", key); - LOG.debug(itemMsg); - out.append(itemMsg).append(System.lineSeparator()); - } - } - } - if (!changedProperties.isEmpty()) { - config.updateProperties(changedProperties); - config.save(); - } - } - } - - return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", out.toString(), ""); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/fc93b279/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index ac0f767..419bb5a 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -159,12 +159,12 @@ </execute-stage> </group> - <group xsi:type="cluster" name="CHANGE_STACK_ROOT_DIRECTORY_REFERENCES" title="Change stack root directory references"> + <group xsi:type="cluster" name="CHANGE_STACK_REFERENCES" title="Change stack references"> <direction>UPGRADE</direction> <skippable>false</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - <execute-stage title="Change stack root directory references"> - <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ChangeStackRootDirectoryAction"/> + <execute-stage title="Change stack references"> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ChangeStackReferencesAction"/> </execute-stage> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/fc93b279/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index be54f84..c4737c9 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -159,12 +159,12 @@ </execute-stage> </group> - <group xsi:type="cluster" name="CHANGE_STACK_ROOT_DIRECTORY_REFERENCES" title="Change stack root directory references"> + <group xsi:type="cluster" name="CHANGE_STACK_REFERENCES" title="Change stack references"> <direction>UPGRADE</direction> <skippable>false</skippable> <supports-auto-skip-failure>false</supports-auto-skip-failure> - <execute-stage title="Change stack root directory references"> - <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ChangeStackRootDirectoryAction"/> + <execute-stage title="Change stack references"> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ChangeStackReferencesAction"/> </execute-stage> </group> http://git-wip-us.apache.org/repos/asf/ambari/blob/fc93b279/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesActionTest.java new file mode 100644 index 0000000..592a95f --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackReferencesActionTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.serveraction.upgrades; + +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.powermock.api.easymock.PowerMock.verifyAll; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.DesiredConfig; +import org.junit.Test; + +import com.google.common.collect.Maps; +import com.google.inject.Injector; + +public class ChangeStackReferencesActionTest { + + @Test + public void testExecute() throws Exception { + String clusterName = "c1"; + String configType = "cluster-env"; + + Injector injector = createNiceMock(Injector.class); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + + Map<String, String> commandParams = Maps.newHashMap(); + commandParams.put("clusterName", clusterName); + + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName(clusterName); + + HostRoleCommand hrc = createNiceMock(HostRoleCommand.class); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)); + + // it's difficult to set up a real ConfigImpl, so use a mock + Config clusterEnv = createNiceMock(Config.class); + expect(clusterEnv.getType()).andReturn(configType).anyTimes(); + + Map<String, String> originalProperties = Maps.newHashMap(); + originalProperties.put("mapreduce_tar_source", "/usr/iop/current/hadoop-client/mapreduce.tar.gz"); + originalProperties.put("pig_tar_destination_folder", "hdfs:///iop/apps/{{ stack_version }}/pig/"); + originalProperties.put("pig_tar_source", "/usr/iop/current/pig-client/pig.tar.gz"); + expect(clusterEnv.getProperties()).andReturn(originalProperties).anyTimes(); + + // this is the crux of the test + Map<String, String> updatedProperties = Maps.newHashMap(); + updatedProperties.put("mapreduce_tar_source", "/usr/hdp/current/hadoop-client/mapreduce.tar.gz"); + updatedProperties.put("pig_tar_destination_folder", "hdfs:///hdp/apps/{{ stack_version }}/pig/"); + updatedProperties.put("pig_tar_source", "/usr/hdp/current/pig-client/pig.tar.gz"); + clusterEnv.updateProperties(updatedProperties); expectLastCall(); + + Map<String, DesiredConfig> desiredConfigs = Collections.singletonMap(configType, createNiceMock(DesiredConfig.class)); + expect(cluster.getDesiredConfigs()).andReturn(desiredConfigs); + expect(cluster.getDesiredConfigByType(configType)).andReturn(clusterEnv).atLeastOnce(); + expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes(); + expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce(); + + ChangeStackReferencesAction underTest = new ChangeStackReferencesAction(); + underTest.setExecutionCommand(executionCommand); + underTest.setHostRoleCommand(hrc); + + Field clustersField = ChangeStackReferencesAction.class.getDeclaredField("clusters"); + clustersField.setAccessible(true); + clustersField.set(underTest, clusters); + + replay(injector, clusters, cluster, clusterEnv, hrc); + + ConcurrentMap<String, Object> emptyMap = Maps.newConcurrentMap(); + underTest.execute(emptyMap); + + verifyAll(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fc93b279/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryActionTest.java deleted file mode 100644 index ceac632..0000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ChangeStackRootDirectoryActionTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.serveraction.upgrades; - -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.replay; -import static org.powermock.api.easymock.PowerMock.verifyAll; - -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; - -import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; -import org.apache.ambari.server.actionmanager.HostRoleCommand; -import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.DesiredConfig; -import org.junit.Test; - -import com.google.common.collect.Maps; -import com.google.inject.Injector; - -public class ChangeStackRootDirectoryActionTest { - - @Test - public void testExecute() throws Exception { - String clusterName = "c1"; - String configType = "cluster-env"; - - Injector injector = createNiceMock(Injector.class); - Clusters clusters = createNiceMock(Clusters.class); - Cluster cluster = createNiceMock(Cluster.class); - - Map<String, String> commandParams = Maps.newHashMap(); - commandParams.put("clusterName", clusterName); - - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName(clusterName); - - HostRoleCommand hrc = createNiceMock(HostRoleCommand.class); - expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)); - - // it's difficult to set up a real ConfigImpl, so use a mock - Config clusterEnv = createNiceMock(Config.class); - expect(clusterEnv.getType()).andReturn(configType).anyTimes(); - - Map<String, String> originalProperties = Maps.newHashMap(); - originalProperties.put("mapreduce_tar_source", "/usr/iop/current/hadoop-client/mapreduce.tar.gz"); - originalProperties.put("pig_tar_destination_folder", "hdfs:///iop/apps/{{ stack_version }}/pig/"); - originalProperties.put("pig_tar_source", "/usr/iop/current/pig-client/pig.tar.gz"); - expect(clusterEnv.getProperties()).andReturn(originalProperties).anyTimes(); - - // this is the crux of the test - Map<String, String> updatedProperties = Maps.newHashMap(); - updatedProperties.put("mapreduce_tar_source", "/usr/hdp/current/hadoop-client/mapreduce.tar.gz"); - updatedProperties.put("pig_tar_source", "/usr/hdp/current/pig-client/pig.tar.gz"); - clusterEnv.updateProperties(updatedProperties); expectLastCall(); - - Map<String, DesiredConfig> desiredConfigs = Collections.singletonMap(configType, createNiceMock(DesiredConfig.class)); - expect(cluster.getDesiredConfigs()).andReturn(desiredConfigs); - expect(cluster.getDesiredConfigByType(configType)).andReturn(clusterEnv).atLeastOnce(); - expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes(); - expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce(); - - ChangeStackRootDirectoryAction underTest = new ChangeStackRootDirectoryAction(); - underTest.setExecutionCommand(executionCommand); - underTest.setHostRoleCommand(hrc); - - Field clustersField = ChangeStackRootDirectoryAction.class.getDeclaredField("clusters"); - clustersField.setAccessible(true); - clustersField.set(underTest, clusters); - - replay(injector, clusters, cluster, clusterEnv, hrc); - - ConcurrentMap<String, Object> emptyMap = Maps.newConcurrentMap(); - underTest.execute(emptyMap); - - verifyAll(); - } -}
